From 0c7bdc60f16a8f90fc3ca06ce07a4781b0f6dc84 Mon Sep 17 00:00:00 2001 From: motatoes Date: Sat, 15 Nov 2025 14:46:43 -0800 Subject: [PATCH 01/12] checkpoint --- backend/controllers/projects_helpers.go | 79 ++++++++++++++++++++++++- libs/ci/github/github.go | 45 ++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index 9a3442db2..b899c1ba6 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -3,11 +3,14 @@ package controllers import ( "encoding/json" "fmt" + "log/slog" + "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/ci" + "github.com/diggerhq/digger/libs/ci/github" "github.com/diggerhq/digger/libs/digger_config" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" - "log/slog" ) func UpdateCheckStatusForBatch(gh utils.GithubClientProvider, batch *models.DiggerBatch) error { @@ -76,6 +79,80 @@ func UpdateCheckStatusForBatch(gh utils.GithubClientProvider, batch *models.Digg return nil } + +func UpdateCheckStatusForBatchWithModernChecks(gh utils.GithubClientProvider, batch *models.DiggerBatch) error { + slog.Info("Updating PR status for batch", + "batchId", batch.ID, + "prNumber", batch.PrNumber, + "batchStatus", batch.Status, + "batchType", batch.BatchType, + ) + + if batch.VCS != models.DiggerVCSGithub { + return fmt.Errorf("We only support github VCS for modern checks at the moment") + } + prService, err := utils.GetPrServiceFromBatch(batch, gh) + if err != nil { + slog.Error("Error getting PR service", + "batchId", batch.ID, + "error", err, + ) + return fmt.Errorf("error getting github service: %v", err) + } + + ghPrService := prService.(github.GithubService) + + + diggerYmlString := batch.DiggerConfig + diggerConfigYml, err := digger_config.LoadDiggerConfigYamlFromString(diggerYmlString) + if err != nil { + slog.Error("Error loading Digger config from batch", + "batchId", batch.ID, + "error", err, + ) + return fmt.Errorf("error loading digger config from batch: %v", err) + } + + config, _, err := digger_config.ConvertDiggerYamlToConfig(diggerConfigYml) + if err != nil { + slog.Error("Error converting Digger YAML to config", + "batchId", batch.ID, + "error", err, + ) + return fmt.Errorf("error converting Digger YAML to config: %v", err) + } + + disableDiggerApplyStatusCheck := config.DisableDiggerApplyStatusCheck + + isPlanBatch := batch.BatchType == orchestrator_scheduler.DiggerCommandPlan + + serializedBatch, err := batch.MapToJsonStruct() + if err != nil { + slog.Error("Error mapping batch to json struct", + "batchId", batch.ID, + "error", err, + ) + return fmt.Errorf("error mapping batch to json struct: %v", err) + } + slog.Debug("Updating PR status for batch", + "batchId", batch.ID, "prNumber", batch.PrNumber, "batchStatus", batch.Status, "batchType", batch.BatchType, + "newStatus", serializedBatch.ToStatusCheck()) + if isPlanBatch { + prService.SetStatus(batch.PrNumber, serializedBatch.ToStatusCheck(), "digger/plan") + ghPrService.CreateCheckRun(name, status, conclusion, title , summary, text,batch.CommitSha) + if disableDiggerApplyStatusCheck == false { + prService.SetStatus(batch.PrNumber, "pending", "digger/apply") + } + + } else { + prService.SetStatus(batch.PrNumber, "success", "digger/plan") + if disableDiggerApplyStatusCheck == false { + prService.SetStatus(batch.PrNumber, serializedBatch.ToStatusCheck(), "digger/apply") + } + } + return nil +} + func UpdateCheckStatusForJob(gh utils.GithubClientProvider, job *models.DiggerJob) error { batch := job.Batch slog.Info("Updating PR status for job", diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index 6c5e59028..e0e0109a6 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -308,6 +308,51 @@ func (svc GithubService) SetStatus(prNumber int, status string, statusContext st return err } +// modern check runs for github (not the commit status) +func (svc GithubService) CreateCheckRun(name string, status string, conclusion string, title string, summary string, text string, headSHA string) (*github.CheckRun, error) { + client := svc.Client + owner := svc.Owner + repoName := svc.RepoName + opts := github.CreateCheckRunOptions{ + Name: name, + HeadSHA: headSHA, // commit SHA to attach the check to + Status: github.String(status), // or "queued" / "in_progress" + Conclusion: github.String(conclusion), // "success", "failure", "neutral", etc. + Output: &github.CheckRunOutput{ + Title: github.String(title), + Summary: github.String(summary), + Text: github.String(text), + }, + } + + ctx := context.Background() + checkRun, _, err := client.Checks.CreateCheckRun(ctx, owner, repoName, opts) + return checkRun, err +} + +func (svc GithubService) UpdateCheckRun(checkRunId string, status string, conclusion string, title string, summary string, text string) (*github.CheckRun, error) { + client := svc.Client + owner := svc.Owner + repoName := svc.RepoName + + checkRunIdInt64, err := strconv.ParseInt(checkRunId, 10, 64) + if err != nil { + return nil, fmt.Errorf("could not convert id %v to i64: %v", checkRunId, err) + } + opts := github.UpdateCheckRunOptions{ + Status: github.String(status), + Conclusion: github.String(conclusion), + Output: &github.CheckRunOutput{ + Title: github.String(title), + Summary: github.String(summary), + Text: github.String(text), + }, + } + ctx := context.Background() + checkRun, _, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts) + return checkRun, err +} + func (svc GithubService) GetCombinedPullRequestStatus(prNumber int) (string, error) { pr, _, err := svc.Client.PullRequests.Get(context.Background(), svc.Owner, svc.RepoName, prNumber) if err != nil { From c164f5cbcea3b038b947e0a7905d27c16875eabe Mon Sep 17 00:00:00 2001 From: motatoes Date: Sat, 15 Nov 2025 19:13:33 -0800 Subject: [PATCH 02/12] checkpoint --- backend/controllers/github_comment.go | 4 +- backend/controllers/github_pull_request.go | 5 +- backend/controllers/projects_helpers.go | 6 +- backend/utils/github.go | 76 +++++++++++++++++++++- ee/backend/controllers/bitbucket.go | 4 +- ee/backend/controllers/gitlab.go | 8 +-- libs/ci/github/github.go | 5 +- 7 files changed, 92 insertions(+), 16 deletions(-) diff --git a/backend/controllers/github_comment.go b/backend/controllers/github_comment.go index 288fafa42..73b36f9dc 100644 --- a/backend/controllers/github_comment.go +++ b/backend/controllers/github_comment.go @@ -492,14 +492,14 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu "command", *diggerCommand, ) // This one is for aggregate reporting - err = utils.SetPRStatusForJobs(ghService, issueNumber, jobs) + err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) return nil } // If we reach here then we have created a comment that would have led to more events segment.Track(*org, repoOwner, vcsActorID, "github", "issue_digger_comment", map[string]string{"comment": commentBody}) - err = utils.SetPRStatusForJobs(ghService, issueNumber, jobs) + err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) if err != nil { slog.Error("Error setting status for PR", "issueNumber", issueNumber, diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index 70e31a221..d413f67cc 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -200,7 +200,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR // This one is for aggregate reporting commentReporterManager.UpdateComment(":construction_worker: No projects impacted") } - err = utils.SetPRStatusForJobs(ghService, prNumber, jobsForImpactedProjects) + err = utils.SetPRCommitStatusForJobs(ghService, prNumber, jobsForImpactedProjects) return nil } @@ -376,7 +376,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR return fmt.Errorf("error initializing comment reporter") } - err = utils.SetPRStatusForJobs(ghService, prNumber, jobsForImpactedProjects) + //err = utils.SetPRCommitStatusForJobs(ghService, prNumber, jobsForImpactedProjects) + err = utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha) if err != nil { slog.Error("Error setting status for PR", "prNumber", prNumber, diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index b899c1ba6..2f09caf34 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -7,8 +7,6 @@ import ( "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" - "github.com/diggerhq/digger/libs/ci" - "github.com/diggerhq/digger/libs/ci/github" "github.com/diggerhq/digger/libs/digger_config" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" ) @@ -100,7 +98,7 @@ func UpdateCheckStatusForBatchWithModernChecks(gh utils.GithubClientProvider, ba return fmt.Errorf("error getting github service: %v", err) } - ghPrService := prService.(github.GithubService) + //ghPrService := prService.(github.GithubService) diggerYmlString := batch.DiggerConfig @@ -139,7 +137,7 @@ func UpdateCheckStatusForBatchWithModernChecks(gh utils.GithubClientProvider, ba "newStatus", serializedBatch.ToStatusCheck()) if isPlanBatch { prService.SetStatus(batch.PrNumber, serializedBatch.ToStatusCheck(), "digger/plan") - ghPrService.CreateCheckRun(name, status, conclusion, title , summary, text,batch.CommitSha) + //ghPrService.CreateCheckRun(name, status, conclusion, title , summary, text,batch.CommitSha) if disableDiggerApplyStatusCheck == false { prService.SetStatus(batch.PrNumber, "pending", "digger/apply") } diff --git a/backend/utils/github.go b/backend/utils/github.go index d00cff511..dcfd350d2 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -169,7 +169,7 @@ func GetGithubService(gh GithubClientProvider, installationId int64, repoFullNam return &ghService, token, nil } -func SetPRStatusForJobs(prService ci.PullRequestService, prNumber int, jobs []scheduler.Job) error { +func SetPRCommitStatusForJobs(prService ci.PullRequestService, prNumber int, jobs []scheduler.Job) error { slog.Info("Setting PR status for jobs", "prNumber", prNumber, "jobCount", len(jobs), @@ -241,6 +241,80 @@ func SetPRStatusForJobs(prService ci.PullRequestService, prNumber int, jobs []sc return nil } +// Checks are the more modern github way as opposed to "commit status" +// With checks you also get to set a page representing content of the check +func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []scheduler.Job, commitSha string) error { + slog.Info("Setting PR status for jobs", + "prNumber", prNumber, + "jobCount", len(jobs), + ) + + + for _, job := range jobs { + for _, command := range job.Commands { + var err error + switch command { + case "digger plan": + slog.Debug("Setting PR status for plan", + "prNumber", prNumber, + "project", job.ProjectName, + ) + _, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/plan", "in_progress", "", job.GetProjectAlias()+"/plan" , "", job.GetProjectAlias()+"/plan", commitSha) + case "digger apply": + slog.Debug("Setting PR status for apply", + "prNumber", prNumber, + "project", job.ProjectName, + ) + _, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/apply", "in_progress", "", job.GetProjectAlias()+"/plan" , "", job.GetProjectAlias()+"/plan", commitSha) + } + if err != nil { + slog.Error("Failed to set PR status", + "prNumber", prNumber, + "project", job.ProjectName, + "command", command, + "error", err, + ) + return fmt.Errorf("Error setting pr status: %v", err) + } + } + } + + // Report aggregate status for digger/plan or digger/apply + if len(jobs) > 0 { + var err error + if scheduler.IsPlanJobs(jobs) { + slog.Debug("Setting aggregate plan status", "prNumber", prNumber) + _, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "digger/plan" , "", "digger/plan", commitSha) + } else { + slog.Debug("Setting aggregate apply status", "prNumber", prNumber) + _, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "digger/apply" , "", "digger/apply", commitSha) + } + if err != nil { + slog.Error("Failed to set aggregate PR status", + "prNumber", prNumber, + "error", err, + ) + return fmt.Errorf("error setting pr status: %v", err) + } + } else { + slog.Debug("Setting success status for empty job list", "prNumber", prNumber) + _, err := ghService.CreateCheckRun("digger/plan", "completed", "success", "digger/plan" , "", "digger/plan", commitSha) + if err != nil { + slog.Error("Failed to set success plan status", "prNumber", prNumber, "error", err) + return fmt.Errorf("error setting pr status: %v", err) + } + + _, err = ghService.CreateCheckRun("digger/apply", "completed", "success", "digger/apply" , "", "digger/apply", commitSha) + if err != nil { + slog.Error("Failed to set success apply status", "prNumber", prNumber, "error", err) + return fmt.Errorf("error setting pr status: %v", err) + } + } + + slog.Info("Successfully set PR status", "prNumber", prNumber) + return nil +} + func GetGithubHostname() string { githubHostname := os.Getenv("DIGGER_GITHUB_HOSTNAME") if githubHostname == "" { diff --git a/ee/backend/controllers/bitbucket.go b/ee/backend/controllers/bitbucket.go index 1183b2569..887ac7f5b 100644 --- a/ee/backend/controllers/bitbucket.go +++ b/ee/backend/controllers/bitbucket.go @@ -278,11 +278,11 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa if len(jobs) == 0 { log.Printf("no projects impacated, succeeding") // This one is for aggregate reporting - err = utils.SetPRStatusForJobs(bbService, issueNumber, jobs) + err = utils.SetPRCommitStatusForJobs(bbService, issueNumber, jobs) return nil } - err = utils.SetPRStatusForJobs(bbService, issueNumber, jobs) + err = utils.SetPRCommitStatusForJobs(bbService, issueNumber, jobs) if err != nil { log.Printf("error setting status for PR: %v", err) utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: error setting status for PR: %v", err)) diff --git a/ee/backend/controllers/gitlab.go b/ee/backend/controllers/gitlab.go index 5027ec036..c0771997c 100644 --- a/ee/backend/controllers/gitlab.go +++ b/ee/backend/controllers/gitlab.go @@ -203,7 +203,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab // TODO use status checks instead: https://github.com/diggerhq/digger/issues/1135 log.Printf("No projects impacted; not starting any jobs") // This one is for aggregate reporting - err = utils.SetPRStatusForJobs(glService, prNumber, jobsForImpactedProjects) + err = utils.SetPRCommitStatusForJobs(glService, prNumber, jobsForImpactedProjects) return nil } @@ -265,7 +265,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab return fmt.Errorf("failed to comment initial status for jobs") } - err = utils.SetPRStatusForJobs(glService, prNumber, jobsForImpactedProjects) + err = utils.SetPRCommitStatusForJobs(glService, prNumber, jobsForImpactedProjects) if err != nil { log.Printf("error setting status for PR: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: error setting status for PR: %v", err)) @@ -461,11 +461,11 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla if len(jobs) == 0 { log.Printf("no projects impacated, succeeding") // This one is for aggregate reporting - err = utils.SetPRStatusForJobs(glService, issueNumber, jobs) + err = utils.SetPRCommitStatusForJobs(glService, issueNumber, jobs) return nil } - err = utils.SetPRStatusForJobs(glService, issueNumber, jobs) + err = utils.SetPRCommitStatusForJobs(glService, issueNumber, jobs) if err != nil { log.Printf("error setting status for PR: %v", err) utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: error setting status for PR: %v", err)) diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index e0e0109a6..1ebf76e74 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -317,7 +317,6 @@ func (svc GithubService) CreateCheckRun(name string, status string, conclusion s Name: name, HeadSHA: headSHA, // commit SHA to attach the check to Status: github.String(status), // or "queued" / "in_progress" - Conclusion: github.String(conclusion), // "success", "failure", "neutral", etc. Output: &github.CheckRunOutput{ Title: github.String(title), Summary: github.String(summary), @@ -325,6 +324,10 @@ func (svc GithubService) CreateCheckRun(name string, status string, conclusion s }, } + if conclusion != "" { + opts.Conclusion = github.String(conclusion) + } + ctx := context.Background() checkRun, _, err := client.Checks.CreateCheckRun(ctx, owner, repoName, opts) return checkRun, err From 493f57885a1e75eec765e88e4d1dedfde13936b2 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 18 Nov 2025 14:27:10 -0800 Subject: [PATCH 03/12] checkpoint --- backend/controllers/github_comment.go | 29 +---- backend/controllers/github_pull_request.go | 4 +- backend/controllers/github_test.go | 8 +- backend/controllers/projects.go | 6 +- backend/controllers/projects_helpers.go | 123 +++++++++++++++++---- backend/migrations/20251118022613.sql | 4 + backend/migrations/atlas.sum | 3 +- backend/models/scheduler.go | 43 ++++++- backend/models/scheduler_test.go | 10 +- backend/models/storage.go | 17 ++- backend/models/storage_test.go | 4 +- backend/tasks/runs_test.go | 2 +- backend/utils/github.go | 38 ++++--- backend/utils/graphs.go | 17 ++- drift/controllers/drift.go | 2 +- ee/backend/controllers/bitbucket.go | 2 +- ee/backend/controllers/gitlab.go | 4 +- ee/backend/hooks/github.go | 2 +- libs/ci/github/github.go | 101 +++++++++++++++-- libs/digger_config/converters.go | 2 +- libs/scheduler/models.go | 34 +++++- 21 files changed, 348 insertions(+), 107 deletions(-) create mode 100644 backend/migrations/20251118022613.sql diff --git a/backend/controllers/github_comment.go b/backend/controllers/github_comment.go index 73b36f9dc..50364aa06 100644 --- a/backend/controllers/github_comment.go +++ b/backend/controllers/github_comment.go @@ -492,14 +492,16 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu "command", *diggerCommand, ) // This one is for aggregate reporting - err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) + //err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) + _, _, err = utils.SetPRCheckForJobs(ghService, issueNumber, jobs, *commitSha) return nil } // If we reach here then we have created a comment that would have led to more events segment.Track(*org, repoOwner, vcsActorID, "github", "issue_digger_comment", map[string]string{"comment": commentBody}) - err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) + //err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) + batchCheckRunId, jobCheckRunIds, err := utils.SetPRCheckForJobs(ghService, issueNumber, jobs, *commitSha) if err != nil { slog.Error("Error setting status for PR", "issueNumber", issueNumber, @@ -557,28 +559,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu "jobCount", len(impactedProjectsJobMap), ) - batchId, _, err := utils.ConvertJobsToDiggerJobs( - *diggerCommand, - "github", - orgId, - impactedProjectsJobMap, - impactedProjectsMap, - projectsGraph, - installationId, - *prSourceBranch, - issueNumber, - repoOwner, - repoName, - repoFullName, - *commitSha, - reporterCommentId, - diggerYmlStr, - 0, - aiSummaryCommentId, - config.ReportTerraformOutputs, - coverAllImpactedProjects, - nil, - ) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *prSourceBranch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, &batchCheckRunId, jobCheckRunIds) if err != nil { slog.Error("Error converting jobs to Digger jobs", "issueNumber", issueNumber, diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index d413f67cc..d31a41edb 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -377,7 +377,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR } //err = utils.SetPRCommitStatusForJobs(ghService, prNumber, jobsForImpactedProjects) - err = utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha) + batchCheckRunId, jobsCheckRunIdsMap, err := utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha) if err != nil { slog.Error("Error setting status for PR", "prNumber", prNumber, @@ -495,6 +495,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR config.ReportTerraformOutputs, coverAllImpactedProjects, nil, + &batchCheckRunId, + jobsCheckRunIdsMap, ) if err != nil { slog.Error("Error converting jobs to Digger jobs", diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go index 5f7219d85..f4c8cabc1 100644 --- a/backend/controllers/github_test.go +++ b/backend/controllers/github_test.go @@ -724,7 +724,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) { graph, err := configuration.CreateProjectDependencyGraph(projects) assert.NoError(t, err) - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, true, nil) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 1, len(result)) @@ -754,7 +754,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -788,7 +788,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -834,7 +834,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) { projectMap["555"] = project5 projectMap["666"] = project6 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 6, len(result)) diff --git a/backend/controllers/projects.go b/backend/controllers/projects.go index e0cde5e4a..27e34376f 100644 --- a/backend/controllers/projects.go +++ b/backend/controllers/projects.go @@ -1006,7 +1006,8 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed batch"}) return } - err = UpdateCheckStatusForBatch(d.GithubClientProvider, refreshedBatch) + //err = UpdateCheckStatusForBatch(d.GithubClientProvider, refreshedBatch) + err = UpdateCheckRunForBatch(d.GithubClientProvider, refreshedBatch) if err != nil { slog.Error("Error updating check status", "batchId", batch.ID, @@ -1026,7 +1027,8 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed job"}) return } - err = UpdateCheckStatusForJob(d.GithubClientProvider, refreshedJob) + //err = UpdateCommitStatusForJob(d.GithubClientProvider, refreshedJob) + err = UpdateCheckRunForJob(d.GithubClientProvider, refreshedJob) if err != nil { slog.Error("Error updating check status", "jobId", jobId, diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index 2f09caf34..ab3886d03 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -7,11 +7,12 @@ import ( "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/ci/github" "github.com/diggerhq/digger/libs/digger_config" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" ) -func UpdateCheckStatusForBatch(gh utils.GithubClientProvider, batch *models.DiggerBatch) error { +func UpdateCommitStatusForBatch(gh utils.GithubClientProvider, batch *models.DiggerBatch) error { slog.Info("Updating PR status for batch", "batchId", batch.ID, "prNumber", batch.PrNumber, @@ -61,9 +62,9 @@ func UpdateCheckStatusForBatch(gh utils.GithubClientProvider, batch *models.Digg } slog.Debug("Updating PR status for batch", "batchId", batch.ID, "prNumber", batch.PrNumber, "batchStatus", batch.Status, "batchType", batch.BatchType, - "newStatus", serializedBatch.ToStatusCheck()) + "newStatus", serializedBatch.ToCommitStatusCheck()) if isPlanBatch { - prService.SetStatus(batch.PrNumber, serializedBatch.ToStatusCheck(), "digger/plan") + prService.SetStatus(batch.PrNumber, serializedBatch.ToCommitStatusCheck(), "digger/plan") if disableDiggerApplyStatusCheck == false { prService.SetStatus(batch.PrNumber, "pending", "digger/apply") } @@ -71,14 +72,13 @@ func UpdateCheckStatusForBatch(gh utils.GithubClientProvider, batch *models.Digg } else { prService.SetStatus(batch.PrNumber, "success", "digger/plan") if disableDiggerApplyStatusCheck == false { - prService.SetStatus(batch.PrNumber, serializedBatch.ToStatusCheck(), "digger/apply") + prService.SetStatus(batch.PrNumber, serializedBatch.ToCommitStatusCheck(), "digger/apply") } } return nil } - -func UpdateCheckStatusForBatchWithModernChecks(gh utils.GithubClientProvider, batch *models.DiggerBatch) error { +func UpdateCheckRunForBatch(gh utils.GithubClientProvider, batch *models.DiggerBatch) error { slog.Info("Updating PR status for batch", "batchId", batch.ID, "prNumber", batch.PrNumber, @@ -86,6 +86,11 @@ func UpdateCheckStatusForBatchWithModernChecks(gh utils.GithubClientProvider, ba "batchType", batch.BatchType, ) + if batch.CheckRunId == nil { + slog.Error("Error checking run id, found nil", "batchId", batch.ID) + return fmt.Errorf("error checking run id, found nil batch") + } + if batch.VCS != models.DiggerVCSGithub { return fmt.Errorf("We only support github VCS for modern checks at the moment") } @@ -98,9 +103,7 @@ func UpdateCheckStatusForBatchWithModernChecks(gh utils.GithubClientProvider, ba return fmt.Errorf("error getting github service: %v", err) } - //ghPrService := prService.(github.GithubService) - - + ghPrService := prService.(*github.GithubService) diggerYmlString := batch.DiggerConfig diggerConfigYml, err := digger_config.LoadDiggerConfigYamlFromString(diggerYmlString) if err != nil { @@ -134,24 +137,34 @@ func UpdateCheckStatusForBatchWithModernChecks(gh utils.GithubClientProvider, ba } slog.Debug("Updating PR status for batch", "batchId", batch.ID, "prNumber", batch.PrNumber, "batchStatus", batch.Status, "batchType", batch.BatchType, - "newStatus", serializedBatch.ToStatusCheck()) - if isPlanBatch { - prService.SetStatus(batch.PrNumber, serializedBatch.ToStatusCheck(), "digger/plan") - //ghPrService.CreateCheckRun(name, status, conclusion, title , summary, text,batch.CommitSha) - if disableDiggerApplyStatusCheck == false { - prService.SetStatus(batch.PrNumber, "pending", "digger/apply") - } + "newStatus", serializedBatch.ToCheckRunStatus()) + + jobs, err := models.DB.GetDiggerJobsForBatch(batch.ID) + if err != nil { + slog.Error("Error getting jobs for batch", + "batchId", batch.ID, + "error", err) + return fmt.Errorf("error getting jobs for batch: %v", err) + } + message, err := utils.GenerateRealtimeCommentMessage(jobs, batch.BatchType) + if err != nil { + slog.Error("Error generating realtime comment message", + "batchId", batch.ID, + "error", err) + return fmt.Errorf("error generating realtime comment message: %v", err) + } + if isPlanBatch { + ghPrService.UpdateCheckRun(*batch.CheckRunId, serializedBatch.ToCheckRunStatus(), serializedBatch.ToCheckRunConclusion(), "1/1 planned", "summary goes here", message) } else { - prService.SetStatus(batch.PrNumber, "success", "digger/plan") if disableDiggerApplyStatusCheck == false { - prService.SetStatus(batch.PrNumber, serializedBatch.ToStatusCheck(), "digger/apply") + ghPrService.UpdateCheckRun(*batch.CheckRunId, serializedBatch.ToCheckRunStatus(), serializedBatch.ToCheckRunConclusion(), "1/1 applied", "summary goes here", message) } } return nil } -func UpdateCheckStatusForJob(gh utils.GithubClientProvider, job *models.DiggerJob) error { +func UpdateCommitStatusForJob(gh utils.GithubClientProvider, job *models.DiggerJob) error { batch := job.Batch slog.Info("Updating PR status for job", "jobId", job.DiggerJobID, @@ -177,7 +190,7 @@ func UpdateCheckStatusForJob(gh utils.GithubClientProvider, job *models.DiggerJo } isPlan := jobSpec.IsPlan() - status, err := models.GetStatusCheckForJob(job) + status, err := models.GetCommitStatusForJob(job) if err != nil { return fmt.Errorf("could not get status check for job: %v", err) } @@ -186,8 +199,76 @@ func UpdateCheckStatusForJob(gh utils.GithubClientProvider, job *models.DiggerJo prService.SetStatus(batch.PrNumber, status, jobSpec.GetProjectAlias()+"/plan") prService.SetStatus(batch.PrNumber, "neutral", jobSpec.GetProjectAlias()+"/apply") } else { - //prService.SetStatus(batch.PrNumber, "success", jobSpec.GetProjectAlias()+"/plan") prService.SetStatus(batch.PrNumber, status, jobSpec.GetProjectAlias()+"/apply") } return nil } + +// more modern check runs on github have their own page +func UpdateCheckRunForJob(gh utils.GithubClientProvider, job *models.DiggerJob) error { + batch := job.Batch + slog.Info("Updating PR Check run for job", + "jobId", job.DiggerJobID, + "prNumber", batch.PrNumber, + "jobStatus", job.Status, + "batchType", batch.BatchType, + ) + + if batch.VCS != models.DiggerVCSGithub { + slog.Error("Error updating PR status for job only github is supported", "batchid", batch.ID, "vcs", batch.VCS) + return fmt.Errorf("Error updating PR status for job only github is supported") + } + + if job.CheckRunId == nil { + slog.Error("Error updating PR status, could not find checkRunId in job", "diggerJobId", job.DiggerJobID) + return fmt.Errorf("Error updating PR status, could not find checkRunId in job") + } + + prService, err := utils.GetPrServiceFromBatch(batch, gh) + ghService := prService.(*github.GithubService) + + if err != nil { + slog.Error("Error getting PR service", + "batchId", batch.ID, + "error", err, + ) + return fmt.Errorf("error getting github service: %v", err) + } + + var jobSpec orchestrator_scheduler.JobJson + err = json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec) + if err != nil { + slog.Error("Could not unmarshal job spec", "jobId", job.DiggerJobID, "error", err) + return fmt.Errorf("could not unmarshal json string: %v", err) + } + + isPlan := jobSpec.IsPlan() + status, err := models.GetCheckRunStatusForJob(job) + if err != nil { + return fmt.Errorf("could not get status check for job: %v", err) + } + + conclusion, err := models.GetCheckRunConclusionForJob(job) + if err != nil { + return fmt.Errorf("could not get conclusion for job: %v", err) + } + + text := "" + + "```terraform\n" + + job.TerraformOutput + + "```\n" + + title := fmt.Sprintf("%v created %v updated %v deleted", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted) + + slog.Debug("Updating PR status for job", "jobId", job.DiggerJobID, "status", status, "conclusion", conclusion) + if isPlan { + _, err = ghService.UpdateCheckRun(*job.CheckRunId, status, conclusion, title, "", text) + if err != nil { + slog.Error("Error updating PR status for job", "error", err) + } + } else { + _, err = ghService.UpdateCheckRun(*job.CheckRunId, status, conclusion, title, "", text) + slog.Error("Error updating PR status for job", "error", err) + } + return nil +} diff --git a/backend/migrations/20251118022613.sql b/backend/migrations/20251118022613.sql new file mode 100644 index 000000000..667c67f32 --- /dev/null +++ b/backend/migrations/20251118022613.sql @@ -0,0 +1,4 @@ +-- Modify "digger_batches" table +ALTER TABLE "public"."digger_batches" ADD COLUMN "check_run_id" text NULL; +-- Modify "digger_jobs" table +ALTER TABLE "public"."digger_jobs" ADD COLUMN "check_run_id" text NULL; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index b92355cf9..a542977a4 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:HUFvDp6jvx9L32hqunXbuvbX3cWvhFy8oVuq3SZ7xzY= +h1:Xh76GXxgaSadhYEwyPxPiqHsHX4GwZtawOTI0PzPbRE= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -69,3 +69,4 @@ h1:HUFvDp6jvx9L32hqunXbuvbX3cWvhFy8oVuq3SZ7xzY= 20251107000100.sql h1:b3USfhlLulZ+6iL9a66Ddpy6uDcYmmyDGZLYzbEjuRA= 20251114205312.sql h1:RBQdD8zLKavCEfZOW8S2r31QBC9ZznCjB1Tw4SDJQGg= 20251114230419.sql h1:/WA7vp7SKgdfe3KHS65nbwE4RUyUmlUOxuQ8tZZ/FQI= +20251118022613.sql h1:nSMv/SJ6gUdKjWWVJJhgPTN18SQrhkkknBJGnx8lhKc= diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 8c1bb422d..9ed1ed5ca 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -14,8 +14,8 @@ import ( type ImpactedProject struct { gorm.Model ID uuid.UUID `gorm:"primary_key"` - RepoFullName string `gorm:"index:idx_org_repo"` - CommitSha string `gorm:"index:idx_org_repo"` + RepoFullName string `gorm:"index:idx_org_repo"` + CommitSha string `gorm:"index:idx_org_repo"` PrNumber *int Branch *string ProjectName string @@ -41,8 +41,9 @@ type DiggerBatch struct { Layer uint VCS DiggerVCSType PrNumber int - CommitSha string + CommitSha string CommentId *int64 + CheckRunId *string AiSummaryCommentId string Status orchestrator_scheduler.DiggerBatchStatus BranchName string @@ -71,6 +72,7 @@ type DiggerJob struct { BatchID *string `gorm:"index:idx_digger_job_id"` PRCommentUrl string PRCommentId *int64 + CheckRunId *string DiggerJobSummary DiggerJobSummary DiggerJobSummaryID uint SerializedJobSpec []byte @@ -198,7 +200,7 @@ func (b *DiggerBatch) MapToJsonStruct() (orchestrator_scheduler.SerializedBatch, return res, nil } -func GetStatusCheckForJob(job *DiggerJob) (string, error) { +func GetCommitStatusForJob(job *DiggerJob) (string, error) { switch job.Status { case orchestrator_scheduler.DiggerJobStarted: return "pending", nil @@ -213,3 +215,36 @@ func GetStatusCheckForJob(job *DiggerJob) (string, error) { } return "", fmt.Errorf("unknown job status: %v", job.Status) } + + +func GetCheckRunStatusForJob(job *DiggerJob) (string, error) { + switch job.Status { + case orchestrator_scheduler.DiggerJobStarted: + return "in_progress", nil + case orchestrator_scheduler.DiggerJobTriggered: + return "in_progress", nil + case orchestrator_scheduler.DiggerJobCreated: + return "in_progress", nil + case orchestrator_scheduler.DiggerJobSucceeded: + return "completed", nil + case orchestrator_scheduler.DiggerJobFailed: + return "completed", nil + } + return "", fmt.Errorf("unknown job status: %v", job.Status) +} + +func GetCheckRunConclusionForJob(job *DiggerJob) (string, error) { + switch job.Status { + case orchestrator_scheduler.DiggerJobStarted: + return "", nil + case orchestrator_scheduler.DiggerJobTriggered: + return "", nil + case orchestrator_scheduler.DiggerJobCreated: + return "", nil + case orchestrator_scheduler.DiggerJobSucceeded: + return "success", nil + case orchestrator_scheduler.DiggerJobFailed: + return "failure", nil + } + return "", fmt.Errorf("unknown job status: %v", job.Status) +} diff --git a/backend/models/scheduler_test.go b/backend/models/scheduler_test.go index 3516e2e42..99aa0f994 100644 --- a/backend/models/scheduler_test.go +++ b/backend/models/scheduler_test.go @@ -75,7 +75,7 @@ func TestCreateDiggerJob(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml") + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) assert.NoError(t, err) assert.NotNil(t, job) @@ -87,7 +87,7 @@ func TestCreateSingleJob(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml") + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) assert.NoError(t, err) assert.NotNil(t, job) @@ -99,20 +99,20 @@ func TestFindDiggerJobsByParentJobId(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml") + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) parentJobId := job.DiggerJobID assert.NoError(t, err) assert.NotNil(t, job) assert.NotZero(t, job.ID) - job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml") + job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) assert.NoError(t, err) assert.NotNil(t, job) assert.NotZero(t, job.ID) err = database.CreateDiggerJobParentLink(parentJobId, job.DiggerJobID) assert.Nil(t, err) - job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml") + job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) assert.NoError(t, err) assert.NotNil(t, job) err = database.CreateDiggerJobParentLink(parentJobId, job.DiggerJobID) diff --git a/backend/models/storage.go b/backend/models/storage.go index 55d55339f..fef9e6de4 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -866,7 +866,7 @@ func (db *Database) GetDiggerBatch(batchId *uuid.UUID) (*DiggerBatch, error) { return batch, nil } -func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool, coverAllImpactedProjects bool, VCSConnectionId *uint, commitSha string) (*DiggerBatch, error) { +func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool, coverAllImpactedProjects bool, VCSConnectionId *uint, commitSha string, checkRunId *string) (*DiggerBatch, error) { uid := uuid.New() batch := &DiggerBatch{ ID: uid, @@ -879,6 +879,7 @@ func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationI PrNumber: PRNumber, CommitSha: commitSha, CommentId: commentId, + CheckRunId: checkRunId, Status: scheduler.BatchJobCreated, BranchName: branchName, DiggerConfig: diggerConfig, @@ -945,7 +946,7 @@ func (db *Database) UpdateBatchStatus(batch *DiggerBatch) error { return nil } -func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, workflowFile string) (*DiggerJob, error) { +func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, workflowFile string, checkRunId *string) (*DiggerJob, error) { if serializedJob == nil || len(serializedJob) == 0 { return nil, fmt.Errorf("serializedJob can't be empty") } @@ -959,8 +960,16 @@ func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, wor } workflowUrl := "#" - job := &DiggerJob{DiggerJobID: jobId, Status: scheduler.DiggerJobCreated, - BatchID: &batchIdStr, SerializedJobSpec: serializedJob, DiggerJobSummary: *summary, WorkflowRunUrl: &workflowUrl, WorkflowFile: workflowFile} + job := &DiggerJob{ + DiggerJobID: jobId, + Status: scheduler.DiggerJobCreated, + BatchID: &batchIdStr, + CheckRunId: checkRunId, + SerializedJobSpec: serializedJob, + DiggerJobSummary: *summary, + WorkflowRunUrl: &workflowUrl, + WorkflowFile: workflowFile, + } result = db.GormDB.Save(job) if result.Error != nil { return nil, result.Error diff --git a/backend/models/storage_test.go b/backend/models/storage_test.go index a16924bb8..874659fc2 100644 --- a/backend/models/storage_test.go +++ b/backend/models/storage_test.go @@ -143,10 +143,10 @@ func TestGetDiggerJobsForBatchPreloadsSummary(t *testing.T) { resourcesUpdated := uint(2) resourcesDeleted := uint(3) - batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false, true, nil, "") + batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false, true, nil, "", nil) assert.NoError(t, err) - job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml") + job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml", nil) assert.NoError(t, err) job, err = DB.UpdateDiggerJobSummary(job.DiggerJobID, resourcesCreated, resourcesUpdated, resourcesDeleted) diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go index 96669e62b..9194d91c3 100644 --- a/backend/tasks/runs_test.go +++ b/backend/tasks/runs_test.go @@ -139,7 +139,7 @@ func TestThatRunQueueItemMovesFromQueuedToPlanningAfterPickup(t *testing.T) { for i, testParam := range testParameters { ciService := github2.MockCiService{} - batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false, true, nil, "") + batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false, true, nil, "", nil) project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), "", nil, "", false, false) planStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) applyStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) diff --git a/backend/utils/github.go b/backend/utils/github.go index dcfd350d2..a156cd9cf 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -7,6 +7,7 @@ import ( "log/slog" net "net/http" "os" + "strconv" "strings" "time" @@ -243,15 +244,19 @@ func SetPRCommitStatusForJobs(prService ci.PullRequestService, prNumber int, job // Checks are the more modern github way as opposed to "commit status" // With checks you also get to set a page representing content of the check -func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []scheduler.Job, commitSha string) error { +func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []scheduler.Job, commitSha string) (string, map[string]string, error) { + slog.Info("commitSha", "commitsha", commitSha) slog.Info("Setting PR status for jobs", "prNumber", prNumber, "jobCount", len(jobs), + "commitSha", commitSha, ) - + var batchCheckRunId string + var jobCheckRunIds = make(map[string]string) for _, job := range jobs { for _, command := range job.Commands { + var cr *github.CheckRun var err error switch command { case "digger plan": @@ -259,22 +264,24 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc "prNumber", prNumber, "project", job.ProjectName, ) - _, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/plan", "in_progress", "", job.GetProjectAlias()+"/plan" , "", job.GetProjectAlias()+"/plan", commitSha) + cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/plan", "in_progress", "", "Waiting for plan..." , "", "Plan result will appear here", commitSha) + jobCheckRunIds[job.ProjectName] = strconv.FormatInt(*cr.ID, 10) case "digger apply": slog.Debug("Setting PR status for apply", "prNumber", prNumber, "project", job.ProjectName, ) - _, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/apply", "in_progress", "", job.GetProjectAlias()+"/plan" , "", job.GetProjectAlias()+"/plan", commitSha) + cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/apply", "in_progress", "", "Waiting for apply..." , "", "Apply result will appear here", commitSha) + jobCheckRunIds[job.ProjectName] = strconv.FormatInt(*cr.ID, 10) } if err != nil { - slog.Error("Failed to set PR status", + slog.Error("Failed to set job PR status", "prNumber", prNumber, "project", job.ProjectName, "command", command, "error", err, ) - return fmt.Errorf("Error setting pr status: %v", err) + return "", nil, fmt.Errorf("Error setting pr status: %v", err) } } } @@ -282,37 +289,40 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc // Report aggregate status for digger/plan or digger/apply if len(jobs) > 0 { var err error + var cr *github.CheckRun if scheduler.IsPlanJobs(jobs) { slog.Debug("Setting aggregate plan status", "prNumber", prNumber) - _, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "digger/plan" , "", "digger/plan", commitSha) + cr, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "Pending start..." , "", "Planning Summary will appear here", commitSha) + batchCheckRunId = strconv.FormatInt(*cr.ID, 10) } else { slog.Debug("Setting aggregate apply status", "prNumber", prNumber) - _, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "digger/apply" , "", "digger/apply", commitSha) + cr, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "Pending start..." , "", "Apply Summary will appear here", commitSha) + batchCheckRunId = strconv.FormatInt(*cr.ID, 10) } if err != nil { slog.Error("Failed to set aggregate PR status", "prNumber", prNumber, "error", err, ) - return fmt.Errorf("error setting pr status: %v", err) + return "", nil, fmt.Errorf("error setting pr status: %v", err) } } else { slog.Debug("Setting success status for empty job list", "prNumber", prNumber) - _, err := ghService.CreateCheckRun("digger/plan", "completed", "success", "digger/plan" , "", "digger/plan", commitSha) + _, err := ghService.CreateCheckRun("digger/plan", "completed", "success", "No impacted projects" , "Check your configuration and files changed if this is unexpected", "digger/plan", commitSha) if err != nil { slog.Error("Failed to set success plan status", "prNumber", prNumber, "error", err) - return fmt.Errorf("error setting pr status: %v", err) + return "", nil, fmt.Errorf("error setting pr status: %v", err) } - _, err = ghService.CreateCheckRun("digger/apply", "completed", "success", "digger/apply" , "", "digger/apply", commitSha) + _, err = ghService.CreateCheckRun("digger/apply", "completed", "success", "No impacted projects" , "Check your configuration and files changed if this is unexpected", "digger/apply", commitSha) if err != nil { slog.Error("Failed to set success apply status", "prNumber", prNumber, "error", err) - return fmt.Errorf("error setting pr status: %v", err) + return "", nil, fmt.Errorf("error setting pr status: %v", err) } } slog.Info("Successfully set PR status", "prNumber", prNumber) - return nil + return batchCheckRunId, jobCheckRunIds, nil } func GetGithubHostname() string { diff --git a/backend/utils/graphs.go b/backend/utils/graphs.go index c09bb2833..22831be8a 100644 --- a/backend/utils/graphs.go +++ b/backend/utils/graphs.go @@ -15,7 +15,7 @@ import ( ) // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value -func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool, VCSConnectionId *uint) (*uuid.UUID, map[string]*models.DiggerJob, error) { +func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool, VCSConnectionId *uint, batchCheckRunId *string, jobsCheckRunIdsMap map[string]string) (*uuid.UUID, map[string]*models.DiggerJob, error) { slog.Info("Converting jobs to Digger jobs", "jobType", jobType, "vcsType", vcsType, @@ -72,7 +72,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig ) } - batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput, coverAllImpactedProjects, VCSConnectionId, commitSha) + batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput, coverAllImpactedProjects, VCSConnectionId, commitSha, batchCheckRunId) if err != nil { slog.Error("Failed to create batch", "error", err) return nil, nil, fmt.Errorf("failed to create batch: %v", err) @@ -93,10 +93,17 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig } visit := func(value string) bool { + var jobCheckRunId *string = nil + if jobsCheckRunIdsMap != nil { + if v, ok := jobsCheckRunIdsMap[value]; ok { + jobCheckRunId = &v + } else { + jobCheckRunId = nil + } + } if predecessorMap[value] == nil || len(predecessorMap[value]) == 0 { slog.Debug("Processing node with no parents", "projectName", value) - - parentJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile) + parentJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId) if err != nil { slog.Error("Failed to create job", "projectName", value, @@ -133,7 +140,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig parent := edge.Source parentDiggerJob := result[parent] - childJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile) + childJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId) if err != nil { slog.Error("Failed to create child job", "projectName", value, diff --git a/drift/controllers/drift.go b/drift/controllers/drift.go index a4d8de4e3..11300e5b5 100644 --- a/drift/controllers/drift.go +++ b/drift/controllers/drift.go @@ -166,7 +166,7 @@ func (mc MainController) TriggerDriftRunForProject(c *gin.Context) { } - batch, err := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, installationid, repoOwner, repoName, repoFullName, 0, "", branch, scheduler.DiggerCommandPlan, nil, 0, "", true, false, nil, "") + batch, err := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, installationid, repoOwner, repoName, repoFullName, 0, "", branch, scheduler.DiggerCommandPlan, nil, 0, "", true, false, nil, "", nil) if err != nil { log.Printf("error creating the batch: %v", err) c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error creating batch entry")}) diff --git a/ee/backend/controllers/bitbucket.go b/ee/backend/controllers/bitbucket.go index 887ac7f5b..3f6da11db 100644 --- a/ee/backend/controllers/bitbucket.go +++ b/ee/backend/controllers/bitbucket.go @@ -305,7 +305,7 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false, true, vcsConnectionId) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false, true, vcsConnectionId, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/ee/backend/controllers/gitlab.go b/ee/backend/controllers/gitlab.go index c0771997c..29e71dc5f 100644 --- a/ee/backend/controllers/gitlab.go +++ b/ee/backend/controllers/gitlab.go @@ -288,7 +288,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false, coverAllImpactedProjects, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -488,7 +488,7 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false, coverAllImpactedProjects, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/ee/backend/hooks/github.go b/ee/backend/hooks/github.go index bbbbb1b5e..ef8f6114e 100644 --- a/ee/backend/hooks/github.go +++ b/ee/backend/hooks/github.go @@ -152,7 +152,7 @@ var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.Git utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false, coverAllImpactedProjects, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index 1ebf76e74..c3dd2b9a4 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -315,8 +315,8 @@ func (svc GithubService) CreateCheckRun(name string, status string, conclusion s repoName := svc.RepoName opts := github.CreateCheckRunOptions{ Name: name, - HeadSHA: headSHA, // commit SHA to attach the check to - Status: github.String(status), // or "queued" / "in_progress" + HeadSHA: headSHA, // commit SHA to attach the check to + Status: github.String(status), // or "queued" / "in_progress" Output: &github.CheckRunOutput{ Title: github.String(title), Summary: github.String(summary), @@ -327,13 +327,24 @@ func (svc GithubService) CreateCheckRun(name string, status string, conclusion s if conclusion != "" { opts.Conclusion = github.String(conclusion) } - + ctx := context.Background() checkRun, _, err := client.Checks.CreateCheckRun(ctx, owner, repoName, opts) + + slog.Debug("created check run", "checkRunId", *checkRun.ID, "externalId", *checkRun.ExternalID) + return checkRun, err } func (svc GithubService) UpdateCheckRun(checkRunId string, status string, conclusion string, title string, summary string, text string) (*github.CheckRun, error) { + slog.Debug("Updating check run", + "checkRunId", checkRunId, + "status", status, + "conclusion", conclusion, + "title", title, + "summary", summary, + "text", text, + ) client := svc.Client owner := svc.Owner repoName := svc.RepoName @@ -342,17 +353,83 @@ func (svc GithubService) UpdateCheckRun(checkRunId string, status string, conclu if err != nil { return nil, fmt.Errorf("could not convert id %v to i64: %v", checkRunId, err) } + + ctx := context.Background() + + // Fetch existing check run to preserve annotations and other output data + existingCheckRun, _, err := client.Checks.GetCheckRun(ctx, owner, repoName, checkRunIdInt64) + if err != nil { + slog.Warn("Failed to fetch existing check run, proceeding with update anyway", + "checkRunId", checkRunId, + "error", err, + ) + return nil, fmt.Errorf("could not fetch existing check run: %v", err) + } + + // Merge existing output with new output, preserving annotations and images + output := &github.CheckRunOutput{} + if existingCheckRun.Output != nil { + // Preserve existing annotations if they exist + if existingCheckRun.Output.Annotations != nil && len(existingCheckRun.Output.Annotations) > 0 { + output.Annotations = existingCheckRun.Output.Annotations + } + // Preserve existing images if they exist + if existingCheckRun.Output.Images != nil && len(existingCheckRun.Output.Images) > 0 { + output.Images = existingCheckRun.Output.Images + } + } + + // Update with new values (only update if provided and non-empty) + if title != "" { + output.Title = github.String(title) + } else if existingCheckRun.Output != nil && existingCheckRun.Output.Title != nil { + output.Title = existingCheckRun.Output.Title + } + + if summary != "" { + output.Summary = github.String(summary) + } else if existingCheckRun.Output != nil && existingCheckRun.Output.Summary != nil { + output.Summary = existingCheckRun.Output.Summary + } + + if text != "" { + output.Text = github.String(text) + } else if existingCheckRun.Output != nil && existingCheckRun.Output.Text != nil { + output.Text = existingCheckRun.Output.Text + } + opts := github.UpdateCheckRunOptions{ - Status: github.String(status), - Conclusion: github.String(conclusion), - Output: &github.CheckRunOutput{ - Title: github.String(title), - Summary: github.String(summary), - Text: github.String(text), - }, + Name: *existingCheckRun.Name, + Status: github.String(status), + Output: output, } - ctx := context.Background() + + if conclusion != "" { + opts.Conclusion = github.String(conclusion) + } + checkRun, _, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts) + if err != nil { + slog.Error("Failed to update check run", + "inputCheckRunId", checkRunId, + "error", err) + return checkRun, err + } + + // Log both input and output IDs for verification + if checkRun != nil && checkRun.ID != nil { + returnedIdStr := strconv.FormatInt(*checkRun.ID, 10) + slog.Debug("Check run updated", + "inputCheckRunId", checkRunId, + "returnedCheckRunId", returnedIdStr, + "externalId", checkRun.ExternalID, + "status", status, + "conclusion", conclusion) + } else { + slog.Warn("Check run updated but returned nil ID", + "inputCheckRunId", checkRunId) + } + return checkRun, err } @@ -907,4 +984,4 @@ func CheckIfShowProjectsComment(event interface{}) bool { slog.Debug("show-projects comment detected") } return result -} \ No newline at end of file +} diff --git a/libs/digger_config/converters.go b/libs/digger_config/converters.go index 6dffdc694..3d3b58a4b 100644 --- a/libs/digger_config/converters.go +++ b/libs/digger_config/converters.go @@ -231,7 +231,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml) (*DiggerConfig, gra if diggerYaml.ReportTerraformOutputs != nil { diggerConfig.ReportTerraformOutputs = *diggerYaml.ReportTerraformOutputs } else { - diggerConfig.ReportTerraformOutputs = false + diggerConfig.ReportTerraformOutputs = true } diggerConfig.Reporting = copyReporterConfig(diggerYaml.Reporting) diff --git a/libs/scheduler/models.go b/libs/scheduler/models.go index da6f85048..33185ec3d 100644 --- a/libs/scheduler/models.go +++ b/libs/scheduler/models.go @@ -145,7 +145,7 @@ func (b *SerializedBatch) IsApply() (bool, error) { return IsPlanJobSpecs(jobSpecs), nil } -func (b *SerializedBatch) ToStatusCheck() string { +func (b *SerializedBatch) ToCommitStatusCheck() string { switch b.Status { case BatchJobCreated: return "pending" @@ -160,6 +160,38 @@ func (b *SerializedBatch) ToStatusCheck() string { } } + +func (b *SerializedBatch) ToCheckRunStatus() string { + switch b.Status { + case BatchJobCreated: + return "in_progress" + case BatchJobInvalidated: + return "completed" + case BatchJobFailed: + return "completed" + case BatchJobSucceeded: + return "completed" + default: + return "in_progress" + } +} + +func (b *SerializedBatch) ToCheckRunConclusion() string { + switch b.Status { + case BatchJobCreated: + return "" + case BatchJobInvalidated: + return "cancelled" + case BatchJobFailed: + return "failure" + case BatchJobSucceeded: + return "success" + default: + return "" + } +} + + func (s *SerializedJob) ResourcesSummaryString(isPlan bool) string { if !isPlan { return "" From 27fa18c638ceac6f7b2b012d72fa017c8ada401a Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 18 Nov 2025 17:49:41 -0800 Subject: [PATCH 04/12] checkpoint --- backend/controllers/github_comment.go | 4 +- backend/controllers/github_pull_request.go | 4 +- backend/controllers/projects_helpers.go | 118 ++++++++++++++++++++- backend/migrations/20251119004103.sql | 4 + backend/migrations/atlas.sum | 3 +- backend/models/scheduler.go | 2 + backend/models/scheduler_test.go | 10 +- backend/models/storage.go | 5 +- backend/models/storage_test.go | 4 +- backend/tasks/runs_test.go | 2 +- backend/utils/github.go | 37 ++++--- backend/utils/github_types.go | 6 ++ backend/utils/graphs.go | 20 ++-- drift/controllers/drift.go | 2 +- libs/ci/github/github.go | 3 - 15 files changed, 181 insertions(+), 43 deletions(-) create mode 100644 backend/migrations/20251119004103.sql create mode 100644 backend/utils/github_types.go diff --git a/backend/controllers/github_comment.go b/backend/controllers/github_comment.go index 50364aa06..4057dc13d 100644 --- a/backend/controllers/github_comment.go +++ b/backend/controllers/github_comment.go @@ -501,7 +501,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu segment.Track(*org, repoOwner, vcsActorID, "github", "issue_digger_comment", map[string]string{"comment": commentBody}) //err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) - batchCheckRunId, jobCheckRunIds, err := utils.SetPRCheckForJobs(ghService, issueNumber, jobs, *commitSha) + batchCheckRunData, jobCheckRunDataMap, err := utils.SetPRCheckForJobs(ghService, issueNumber, jobs, *commitSha) if err != nil { slog.Error("Error setting status for PR", "issueNumber", issueNumber, @@ -559,7 +559,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu "jobCount", len(impactedProjectsJobMap), ) - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *prSourceBranch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, &batchCheckRunId, jobCheckRunIds) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *prSourceBranch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, batchCheckRunData, jobCheckRunDataMap) if err != nil { slog.Error("Error converting jobs to Digger jobs", "issueNumber", issueNumber, diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index d31a41edb..b0462ed43 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -377,7 +377,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR } //err = utils.SetPRCommitStatusForJobs(ghService, prNumber, jobsForImpactedProjects) - batchCheckRunId, jobsCheckRunIdsMap, err := utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha) + batchCheckRunData, jobsCheckRunIdsMap, err := utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha) if err != nil { slog.Error("Error setting status for PR", "prNumber", prNumber, @@ -495,7 +495,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR config.ReportTerraformOutputs, coverAllImpactedProjects, nil, - &batchCheckRunId, + batchCheckRunData, jobsCheckRunIdsMap, ) if err != nil { diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index ab3886d03..2321046d3 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -4,14 +4,111 @@ import ( "encoding/json" "fmt" "log/slog" + "os" "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/ci" "github.com/diggerhq/digger/libs/ci/github" "github.com/diggerhq/digger/libs/digger_config" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" ) + +func GenerateChecksSummaryForBatch(prService ci.PullRequestService, batch *models.DiggerBatch) (string, error) { + summaryEndpoint := os.Getenv("DIGGER_AI_SUMMARY_ENDPOINT") + if summaryEndpoint == "" { + slog.Error("DIGGER_AI_SUMMARY_ENDPOINT not set") + return"", fmt.Errorf("could not generate AI summary, ai summary endpoint missing") + } + apiToken := os.Getenv("DIGGER_AI_SUMMARY_API_TOKEN") + + jobs, err := models.DB.GetDiggerJobsForBatch(batch.ID) + if err != nil { + slog.Error("Could not get jobs for batch", + "batchId", batch.ID, + "error", err, + ) + + return "", fmt.Errorf("could not get jobs for batch: %v", err) + } + + terraformOutputs := "" + for _, job := range jobs { + var jobSpec orchestrator_scheduler.JobJson + err := json.Unmarshal(job.SerializedJobSpec, &jobSpec) + if err != nil { + slog.Error("Could not unmarshal job spec", + "jobId", job.DiggerJobID, + "error", err, + ) + + return "", fmt.Errorf("could not summarize plans due to unmarshalling error: %v", err) + } + + projectName := jobSpec.ProjectName + slog.Debug("Adding Terraform output for project", + "projectName", projectName, + "jobId", job.DiggerJobID, + "outputLength", len(job.TerraformOutput), + ) + + terraformOutputs += fmt.Sprintf("terraform output for %v: %v \n\n", projectName, job.TerraformOutput) + } + + aiSummary, err := utils.GetAiSummaryFromTerraformPlans(terraformOutputs, summaryEndpoint, apiToken) + if err != nil { + slog.Error("Could not generate AI summary from Terraform outputs", + "batchId", batch.ID, + "error", err, + ) + + return "", fmt.Errorf("could not summarize terraform outputs: %v", err) + } + + summary := "" + if aiSummary != "FOUR_OH_FOUR" { + summary = fmt.Sprintf("**AI summary:** %v", aiSummary) + } + + + return summary, nil +} + +func GenerateChecksSummaryForJob(prService ci.PullRequestService, job *models.DiggerJob) (string, error) { + batch := job.Batch + summaryEndpoint := os.Getenv("DIGGER_AI_SUMMARY_ENDPOINT") + if summaryEndpoint == "" { + slog.Error("AI summary endpoint not configured", "batch", batch.ID, "jobId", job.ID, "DiggerJobId", job.DiggerJobID) + return"", fmt.Errorf("could not generate AI summary, ai summary endpoint missing") + } + apiToken := os.Getenv("DIGGER_AI_SUMMARY_API_TOKEN") + + if job.TerraformOutput == "" { + slog.Warn("Terraform output not set yet, ignoring this call") + return "", nil + } + terraformOutput := fmt.Sprintf("Terraform output for: %v\n\n", job.TerraformOutput) + aiSummary, err := utils.GetAiSummaryFromTerraformPlans(terraformOutput, summaryEndpoint, apiToken) + if err != nil { + slog.Error("Could not generate AI summary from Terraform outputs", + "batchId", batch.ID, + "error", err, + ) + + return "", fmt.Errorf("could not summarize terraform outputs: %v", err) + } + + summary := "" + if aiSummary != "FOUR_OH_FOUR" { + summary = fmt.Sprintf("**AI summary:** %v", aiSummary) + } + return summary, nil +} + + + + func UpdateCommitStatusForBatch(gh utils.GithubClientProvider, batch *models.DiggerBatch) error { slog.Info("Updating PR status for batch", "batchId", batch.ID, @@ -154,11 +251,16 @@ func UpdateCheckRunForBatch(gh utils.GithubClientProvider, batch *models.DiggerB return fmt.Errorf("error generating realtime comment message: %v", err) } + summary, err := GenerateChecksSummaryForBatch(ghPrService, batch) + if err != nil { + slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + } + if isPlanBatch { - ghPrService.UpdateCheckRun(*batch.CheckRunId, serializedBatch.ToCheckRunStatus(), serializedBatch.ToCheckRunConclusion(), "1/1 planned", "summary goes here", message) + ghPrService.UpdateCheckRun(*batch.CheckRunId, serializedBatch.ToCheckRunStatus(), serializedBatch.ToCheckRunConclusion(), "Plans Summary", summary, message) } else { if disableDiggerApplyStatusCheck == false { - ghPrService.UpdateCheckRun(*batch.CheckRunId, serializedBatch.ToCheckRunStatus(), serializedBatch.ToCheckRunConclusion(), "1/1 applied", "summary goes here", message) + ghPrService.UpdateCheckRun(*batch.CheckRunId, serializedBatch.ToCheckRunStatus(), serializedBatch.ToCheckRunConclusion(), "Apply Summary", summary, message) } } return nil @@ -258,16 +360,22 @@ func UpdateCheckRunForJob(gh utils.GithubClientProvider, job *models.DiggerJob) job.TerraformOutput + "```\n" - title := fmt.Sprintf("%v created %v updated %v deleted", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted) + + summary, err := GenerateChecksSummaryForJob(ghService, job) + if err != nil { + slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + } slog.Debug("Updating PR status for job", "jobId", job.DiggerJobID, "status", status, "conclusion", conclusion) if isPlan { - _, err = ghService.UpdateCheckRun(*job.CheckRunId, status, conclusion, title, "", text) + title := fmt.Sprintf("%v to create %v to update %v to delete", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted) + _, err = ghService.UpdateCheckRun(*job.CheckRunId, status, conclusion, title, summary, text) if err != nil { slog.Error("Error updating PR status for job", "error", err) } } else { - _, err = ghService.UpdateCheckRun(*job.CheckRunId, status, conclusion, title, "", text) + title := fmt.Sprintf("%v created %v updated %v deleted", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted) + _, err = ghService.UpdateCheckRun(*job.CheckRunId, status, conclusion, title, summary, text) slog.Error("Error updating PR status for job", "error", err) } return nil diff --git a/backend/migrations/20251119004103.sql b/backend/migrations/20251119004103.sql new file mode 100644 index 000000000..a34906428 --- /dev/null +++ b/backend/migrations/20251119004103.sql @@ -0,0 +1,4 @@ +-- Modify "digger_batches" table +ALTER TABLE "public"."digger_batches" ADD COLUMN "check_run_url" text NULL; +-- Modify "digger_jobs" table +ALTER TABLE "public"."digger_jobs" ADD COLUMN "check_run_url" text NULL; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index a542977a4..095d82242 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:Xh76GXxgaSadhYEwyPxPiqHsHX4GwZtawOTI0PzPbRE= +h1:KHwly+q9MAIOsJSqgLlXD/pJHaVU/ObSzdiaeby9+Tg= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -70,3 +70,4 @@ h1:Xh76GXxgaSadhYEwyPxPiqHsHX4GwZtawOTI0PzPbRE= 20251114205312.sql h1:RBQdD8zLKavCEfZOW8S2r31QBC9ZznCjB1Tw4SDJQGg= 20251114230419.sql h1:/WA7vp7SKgdfe3KHS65nbwE4RUyUmlUOxuQ8tZZ/FQI= 20251118022613.sql h1:nSMv/SJ6gUdKjWWVJJhgPTN18SQrhkkknBJGnx8lhKc= +20251119004103.sql h1:zdyEn54C6mY5iKZ86LQWhOi13sSA2EMriE1lQ9wGi6w= diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 9ed1ed5ca..8e3576cf7 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -44,6 +44,7 @@ type DiggerBatch struct { CommitSha string CommentId *int64 CheckRunId *string + CheckRunUrl *string AiSummaryCommentId string Status orchestrator_scheduler.DiggerBatchStatus BranchName string @@ -73,6 +74,7 @@ type DiggerJob struct { PRCommentUrl string PRCommentId *int64 CheckRunId *string + CheckRunUrl *string DiggerJobSummary DiggerJobSummary DiggerJobSummaryID uint SerializedJobSpec []byte diff --git a/backend/models/scheduler_test.go b/backend/models/scheduler_test.go index 99aa0f994..9cb4675f4 100644 --- a/backend/models/scheduler_test.go +++ b/backend/models/scheduler_test.go @@ -75,7 +75,7 @@ func TestCreateDiggerJob(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) assert.NoError(t, err) assert.NotNil(t, job) @@ -87,7 +87,7 @@ func TestCreateSingleJob(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) assert.NoError(t, err) assert.NotNil(t, job) @@ -99,20 +99,20 @@ func TestFindDiggerJobsByParentJobId(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) parentJobId := job.DiggerJobID assert.NoError(t, err) assert.NotNil(t, job) assert.NotZero(t, job.ID) - job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) + job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) assert.NoError(t, err) assert.NotNil(t, job) assert.NotZero(t, job.ID) err = database.CreateDiggerJobParentLink(parentJobId, job.DiggerJobID) assert.Nil(t, err) - job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil) + job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) assert.NoError(t, err) assert.NotNil(t, job) err = database.CreateDiggerJobParentLink(parentJobId, job.DiggerJobID) diff --git a/backend/models/storage.go b/backend/models/storage.go index fef9e6de4..7da66b9aa 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -866,7 +866,7 @@ func (db *Database) GetDiggerBatch(batchId *uuid.UUID) (*DiggerBatch, error) { return batch, nil } -func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool, coverAllImpactedProjects bool, VCSConnectionId *uint, commitSha string, checkRunId *string) (*DiggerBatch, error) { +func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool, coverAllImpactedProjects bool, VCSConnectionId *uint, commitSha string, checkRunId *string, checkRunUrl *string) (*DiggerBatch, error) { uid := uuid.New() batch := &DiggerBatch{ ID: uid, @@ -880,6 +880,7 @@ func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationI CommitSha: commitSha, CommentId: commentId, CheckRunId: checkRunId, + CheckRunUrl: checkRunUrl, Status: scheduler.BatchJobCreated, BranchName: branchName, DiggerConfig: diggerConfig, @@ -946,7 +947,7 @@ func (db *Database) UpdateBatchStatus(batch *DiggerBatch) error { return nil } -func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, workflowFile string, checkRunId *string) (*DiggerJob, error) { +func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, workflowFile string, checkRunId *string, checkRunUrl *string) (*DiggerJob, error) { if serializedJob == nil || len(serializedJob) == 0 { return nil, fmt.Errorf("serializedJob can't be empty") } diff --git a/backend/models/storage_test.go b/backend/models/storage_test.go index 874659fc2..0b9f847af 100644 --- a/backend/models/storage_test.go +++ b/backend/models/storage_test.go @@ -143,10 +143,10 @@ func TestGetDiggerJobsForBatchPreloadsSummary(t *testing.T) { resourcesUpdated := uint(2) resourcesDeleted := uint(3) - batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false, true, nil, "", nil) + batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false, true, nil, "", nil, nil) assert.NoError(t, err) - job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml", nil) + job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml", nil, nil) assert.NoError(t, err) job, err = DB.UpdateDiggerJobSummary(job.DiggerJobID, resourcesCreated, resourcesUpdated, resourcesDeleted) diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go index 9194d91c3..015b60d4c 100644 --- a/backend/tasks/runs_test.go +++ b/backend/tasks/runs_test.go @@ -139,7 +139,7 @@ func TestThatRunQueueItemMovesFromQueuedToPlanningAfterPickup(t *testing.T) { for i, testParam := range testParameters { ciService := github2.MockCiService{} - batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false, true, nil, "", nil) + batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false, true, nil, "", nil, nil) project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), "", nil, "", false, false) planStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) applyStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) diff --git a/backend/utils/github.go b/backend/utils/github.go index a156cd9cf..ab8314efc 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -244,15 +244,15 @@ func SetPRCommitStatusForJobs(prService ci.PullRequestService, prNumber int, job // Checks are the more modern github way as opposed to "commit status" // With checks you also get to set a page representing content of the check -func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []scheduler.Job, commitSha string) (string, map[string]string, error) { +func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []scheduler.Job, commitSha string) (*CheckRunData, map[string]CheckRunData, error) { slog.Info("commitSha", "commitsha", commitSha) slog.Info("Setting PR status for jobs", "prNumber", prNumber, "jobCount", len(jobs), "commitSha", commitSha, ) - var batchCheckRunId string - var jobCheckRunIds = make(map[string]string) + var batchCheckRunId CheckRunData + var jobCheckRunIds = make(map[string]CheckRunData) for _, job := range jobs { for _, command := range job.Commands { @@ -265,14 +265,21 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc "project", job.ProjectName, ) cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/plan", "in_progress", "", "Waiting for plan..." , "", "Plan result will appear here", commitSha) - jobCheckRunIds[job.ProjectName] = strconv.FormatInt(*cr.ID, 10) + jobCheckRunIds[job.ProjectName] = CheckRunData{ + Id: strconv.FormatInt(*cr.ID, 10), + Url: *cr.URL, + } + case "digger apply": slog.Debug("Setting PR status for apply", "prNumber", prNumber, "project", job.ProjectName, ) cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/apply", "in_progress", "", "Waiting for apply..." , "", "Apply result will appear here", commitSha) - jobCheckRunIds[job.ProjectName] = strconv.FormatInt(*cr.ID, 10) + jobCheckRunIds[job.ProjectName] = CheckRunData{ + Id: strconv.FormatInt(*cr.ID, 10), + Url: *cr.URL, + } } if err != nil { slog.Error("Failed to set job PR status", @@ -281,7 +288,7 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc "command", command, "error", err, ) - return "", nil, fmt.Errorf("Error setting pr status: %v", err) + return nil, nil, fmt.Errorf("Error setting pr status: %v", err) } } } @@ -293,36 +300,42 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc if scheduler.IsPlanJobs(jobs) { slog.Debug("Setting aggregate plan status", "prNumber", prNumber) cr, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "Pending start..." , "", "Planning Summary will appear here", commitSha) - batchCheckRunId = strconv.FormatInt(*cr.ID, 10) + batchCheckRunId = CheckRunData{ + Id: strconv.FormatInt(*cr.ID, 10), + Url: *cr.HTMLURL, + } } else { slog.Debug("Setting aggregate apply status", "prNumber", prNumber) cr, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "Pending start..." , "", "Apply Summary will appear here", commitSha) - batchCheckRunId = strconv.FormatInt(*cr.ID, 10) + batchCheckRunId = CheckRunData{ + Id: strconv.FormatInt(*cr.ID, 10), + Url: *cr.HTMLURL, + } } if err != nil { slog.Error("Failed to set aggregate PR status", "prNumber", prNumber, "error", err, ) - return "", nil, fmt.Errorf("error setting pr status: %v", err) + return nil, nil, fmt.Errorf("error setting pr status: %v", err) } } else { slog.Debug("Setting success status for empty job list", "prNumber", prNumber) _, err := ghService.CreateCheckRun("digger/plan", "completed", "success", "No impacted projects" , "Check your configuration and files changed if this is unexpected", "digger/plan", commitSha) if err != nil { slog.Error("Failed to set success plan status", "prNumber", prNumber, "error", err) - return "", nil, fmt.Errorf("error setting pr status: %v", err) + return nil, nil, fmt.Errorf("error setting pr status: %v", err) } _, err = ghService.CreateCheckRun("digger/apply", "completed", "success", "No impacted projects" , "Check your configuration and files changed if this is unexpected", "digger/apply", commitSha) if err != nil { slog.Error("Failed to set success apply status", "prNumber", prNumber, "error", err) - return "", nil, fmt.Errorf("error setting pr status: %v", err) + return nil, nil, fmt.Errorf("error setting pr status: %v", err) } } slog.Info("Successfully set PR status", "prNumber", prNumber) - return batchCheckRunId, jobCheckRunIds, nil + return &batchCheckRunId, jobCheckRunIds, nil } func GetGithubHostname() string { diff --git a/backend/utils/github_types.go b/backend/utils/github_types.go new file mode 100644 index 000000000..93f3147eb --- /dev/null +++ b/backend/utils/github_types.go @@ -0,0 +1,6 @@ +package utils + +type CheckRunData struct { + Id string + Url string +} \ No newline at end of file diff --git a/backend/utils/graphs.go b/backend/utils/graphs.go index 22831be8a..40a8f6429 100644 --- a/backend/utils/graphs.go +++ b/backend/utils/graphs.go @@ -15,7 +15,7 @@ import ( ) // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value -func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool, VCSConnectionId *uint, batchCheckRunId *string, jobsCheckRunIdsMap map[string]string) (*uuid.UUID, map[string]*models.DiggerJob, error) { +func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool, VCSConnectionId *uint, batchCheckRunData *CheckRunData, jobsCheckRunIdsMap map[string]CheckRunData) (*uuid.UUID, map[string]*models.DiggerJob, error) { slog.Info("Converting jobs to Digger jobs", "jobType", jobType, "vcsType", vcsType, @@ -72,7 +72,13 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig ) } - batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput, coverAllImpactedProjects, VCSConnectionId, commitSha, batchCheckRunId) + var batchCheckRunId *string = nil + var batchCheckRunUrl *string = nil + if batchCheckRunData != nil { + batchCheckRunId = &batchCheckRunData.Id + batchCheckRunUrl = &batchCheckRunData.Url + } + batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput, coverAllImpactedProjects, VCSConnectionId, commitSha, batchCheckRunId, batchCheckRunUrl) if err != nil { slog.Error("Failed to create batch", "error", err) return nil, nil, fmt.Errorf("failed to create batch: %v", err) @@ -94,16 +100,16 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig visit := func(value string) bool { var jobCheckRunId *string = nil + var jobCheckRunUrl *string = nil if jobsCheckRunIdsMap != nil { if v, ok := jobsCheckRunIdsMap[value]; ok { - jobCheckRunId = &v - } else { - jobCheckRunId = nil + jobCheckRunId = &v.Id + jobCheckRunUrl = &v.Url } } if predecessorMap[value] == nil || len(predecessorMap[value]) == 0 { slog.Debug("Processing node with no parents", "projectName", value) - parentJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId) + parentJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl) if err != nil { slog.Error("Failed to create job", "projectName", value, @@ -140,7 +146,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig parent := edge.Source parentDiggerJob := result[parent] - childJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId) + childJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl) if err != nil { slog.Error("Failed to create child job", "projectName", value, diff --git a/drift/controllers/drift.go b/drift/controllers/drift.go index 11300e5b5..5cfaa3e68 100644 --- a/drift/controllers/drift.go +++ b/drift/controllers/drift.go @@ -166,7 +166,7 @@ func (mc MainController) TriggerDriftRunForProject(c *gin.Context) { } - batch, err := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, installationid, repoOwner, repoName, repoFullName, 0, "", branch, scheduler.DiggerCommandPlan, nil, 0, "", true, false, nil, "", nil) + batch, err := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, installationid, repoOwner, repoName, repoFullName, 0, "", branch, scheduler.DiggerCommandPlan, nil, 0, "", true, false, nil, "", nil, nil) if err != nil { log.Printf("error creating the batch: %v", err) c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error creating batch entry")}) diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index c3dd2b9a4..ff5cbd094 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -330,9 +330,6 @@ func (svc GithubService) CreateCheckRun(name string, status string, conclusion s ctx := context.Background() checkRun, _, err := client.Checks.CreateCheckRun(ctx, owner, repoName, opts) - - slog.Debug("created check run", "checkRunId", *checkRun.ID, "externalId", *checkRun.ExternalID) - return checkRun, err } From b75922a37449e37a2b1b5570598a62772856f511 Mon Sep 17 00:00:00 2001 From: motatoes Date: Wed, 19 Nov 2025 09:31:42 -0800 Subject: [PATCH 05/12] checkpoint --- backend/controllers/github_pull_request.go | 1 + backend/controllers/projects_helpers.go | 20 +++++++++------- backend/models/storage.go | 1 + backend/utils/comment_utils.go | 14 +++++++---- backend/utils/github.go | 7 +++--- backend/utils/pr_comment.go | 27 +++++++++++++--------- 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index b0462ed43..12547033e 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -201,6 +201,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR commentReporterManager.UpdateComment(":construction_worker: No projects impacted") } err = utils.SetPRCommitStatusForJobs(ghService, prNumber, jobsForImpactedProjects) + _, _, err = utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha) return nil } diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index 2321046d3..e25c162d0 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -8,14 +8,13 @@ import ( "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" - "github.com/diggerhq/digger/libs/ci" "github.com/diggerhq/digger/libs/ci/github" "github.com/diggerhq/digger/libs/digger_config" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" ) -func GenerateChecksSummaryForBatch(prService ci.PullRequestService, batch *models.DiggerBatch) (string, error) { +func GenerateChecksSummaryForBatch( batch *models.DiggerBatch) (string, error) { summaryEndpoint := os.Getenv("DIGGER_AI_SUMMARY_ENDPOINT") if summaryEndpoint == "" { slog.Error("DIGGER_AI_SUMMARY_ENDPOINT not set") @@ -68,14 +67,13 @@ func GenerateChecksSummaryForBatch(prService ci.PullRequestService, batch *model summary := "" if aiSummary != "FOUR_OH_FOUR" { - summary = fmt.Sprintf("**AI summary:** %v", aiSummary) + summary = fmt.Sprintf(":sparkles: **AI summary:** %v", aiSummary) } - return summary, nil } -func GenerateChecksSummaryForJob(prService ci.PullRequestService, job *models.DiggerJob) (string, error) { +func GenerateChecksSummaryForJob( job *models.DiggerJob) (string, error) { batch := job.Batch summaryEndpoint := os.Getenv("DIGGER_AI_SUMMARY_ENDPOINT") if summaryEndpoint == "" { @@ -100,9 +98,15 @@ func GenerateChecksSummaryForJob(prService ci.PullRequestService, job *models.Di } summary := "" + + if job.WorkflowRunUrl != nil { + summary += fmt.Sprintf(":link: CI job\n\n", *job.WorkflowRunUrl ) + } + if aiSummary != "FOUR_OH_FOUR" { - summary = fmt.Sprintf("**AI summary:** %v", aiSummary) + summary += fmt.Sprintf(":sparkles: **AI summary:** %v", aiSummary) } + return summary, nil } @@ -251,7 +255,7 @@ func UpdateCheckRunForBatch(gh utils.GithubClientProvider, batch *models.DiggerB return fmt.Errorf("error generating realtime comment message: %v", err) } - summary, err := GenerateChecksSummaryForBatch(ghPrService, batch) + summary, err := GenerateChecksSummaryForBatch(batch) if err != nil { slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) } @@ -361,7 +365,7 @@ func UpdateCheckRunForJob(gh utils.GithubClientProvider, job *models.DiggerJob) "```\n" - summary, err := GenerateChecksSummaryForJob(ghService, job) + summary, err := GenerateChecksSummaryForJob(job) if err != nil { slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) } diff --git a/backend/models/storage.go b/backend/models/storage.go index 7da66b9aa..e0f50d087 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -966,6 +966,7 @@ func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, wor Status: scheduler.DiggerJobCreated, BatchID: &batchIdStr, CheckRunId: checkRunId, + CheckRunUrl: checkRunUrl, SerializedJobSpec: serializedJob, DiggerJobSummary: *summary, WorkflowRunUrl: &workflowUrl, diff --git a/backend/utils/comment_utils.go b/backend/utils/comment_utils.go index 78a0b892c..787c302cf 100644 --- a/backend/utils/comment_utils.go +++ b/backend/utils/comment_utils.go @@ -133,9 +133,15 @@ func GenerateRealtimeCommentMessage(jobs []models.DiggerJob, batchType orchestra } // Safe handling of WorkflowRunUrl pointer - workflowUrl := "#" + + var checkRunUrl = "#" + if job.CheckRunUrl != nil { + checkRunUrl = *job.CheckRunUrl + } + + workflowRunUrl := "#" if job.WorkflowRunUrl != nil { - workflowUrl = *job.WorkflowRunUrl + workflowRunUrl = *job.WorkflowRunUrl } // Get project name from job spec @@ -173,9 +179,9 @@ func GenerateRealtimeCommentMessage(jobs []models.DiggerJob, batchType orchestra message += fmt.Sprintf("|%s **%s** |%s | %s | %d | %d | %d|\n", job.Status.ToEmoji(), projectDisplayName, - workflowUrl, + workflowRunUrl, job.Status.ToString(), - prCommentUrl, + checkRunUrl, jobTypeTitle, resourcesCreated, resourcesUpdated, diff --git a/backend/utils/github.go b/backend/utils/github.go index ab8314efc..768f75217 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -267,7 +267,7 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/plan", "in_progress", "", "Waiting for plan..." , "", "Plan result will appear here", commitSha) jobCheckRunIds[job.ProjectName] = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), - Url: *cr.URL, + Url: *cr.HTMLURL, } case "digger apply": @@ -294,19 +294,20 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc } // Report aggregate status for digger/plan or digger/apply + jobsSummaryTable := GetInitialJobSummary(jobs) if len(jobs) > 0 { var err error var cr *github.CheckRun if scheduler.IsPlanJobs(jobs) { slog.Debug("Setting aggregate plan status", "prNumber", prNumber) - cr, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "Pending start..." , "", "Planning Summary will appear here", commitSha) + cr, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "Pending start..." , "", jobsSummaryTable, commitSha) batchCheckRunId = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), Url: *cr.HTMLURL, } } else { slog.Debug("Setting aggregate apply status", "prNumber", prNumber) - cr, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "Pending start..." , "", "Apply Summary will appear here", commitSha) + cr, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "Pending start..." , "", jobsSummaryTable, commitSha) batchCheckRunId = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), Url: *cr.HTMLURL, diff --git a/backend/utils/pr_comment.go b/backend/utils/pr_comment.go index 40400f3b0..ad25a1a54 100644 --- a/backend/utils/pr_comment.go +++ b/backend/utils/pr_comment.go @@ -169,17 +169,7 @@ func ReportInitialJobsStatus(cr *CommentReporter, jobs []scheduler.Job) error { "jobCount", len(jobs), ) - message := "" - if len(jobs) == 0 { - message = message + ":construction_worker: No projects impacted" - } else { - message = message + fmt.Sprintf("| Project | Status |\n") - message = message + fmt.Sprintf("|---------|--------|\n") - for _, job := range jobs { - message = message + fmt.Sprintf(""+ - "|:clock11: **%v**|pending...|\n", job.GetProjectAlias()) - } - } + message := GetInitialJobSummary(jobs) message = trimMessageIfExceedsMaxLength(message) err := prService.EditComment(prNumber, commentId, message) @@ -196,6 +186,21 @@ func ReportInitialJobsStatus(cr *CommentReporter, jobs []scheduler.Job) error { return nil } +func GetInitialJobSummary(jobs []scheduler.Job) string { + message := "" + if len(jobs) == 0 { + message = message + ":construction_worker: No projects impacted" + } else { + message = message + fmt.Sprintf("| Project | Status |\n") + message = message + fmt.Sprintf("|---------|--------|\n") + for _, job := range jobs { + message = message + fmt.Sprintf(""+ + "|:clock11: **%v**|pending...|\n", job.GetProjectAlias()) + } + } + return message +} + func ReportLayersTableForJobs(cr *CommentReporter, jobs []scheduler.Job) error { prNumber := cr.PrNumber prService := cr.PrService From 9340fc8693593b4f96e1315a3a9bafc101aa24fd Mon Sep 17 00:00:00 2001 From: motatoes Date: Wed, 19 Nov 2025 18:19:01 -0800 Subject: [PATCH 06/12] add checkpoint for job --- backend/controllers/github.go | 29 +++++++-- backend/controllers/github_comment.go | 19 +++--- backend/controllers/github_pull_request.go | 20 +++--- backend/controllers/projects_helpers.go | 46 ++++++++++++-- backend/migrations/20251120020911.sql | 2 + backend/migrations/atlas.sum | 3 +- backend/models/scheduler.go | 2 +- backend/models/storage.go | 2 + backend/utils/github.go | 37 +++++++++-- libs/ci/github/github.go | 73 ++++++++++++++-------- libs/scheduler/models.go | 15 +++-- 11 files changed, 182 insertions(+), 66 deletions(-) create mode 100644 backend/migrations/20251120020911.sql diff --git a/backend/controllers/github.go b/backend/controllers/github.go index fe35ad39a..67295ffe6 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -2,6 +2,11 @@ package controllers import ( "context" + "log/slog" + "net/http" + "reflect" + "strconv" + "github.com/diggerhq/digger/backend/ci_backends" "github.com/diggerhq/digger/backend/logging" "github.com/diggerhq/digger/backend/middleware" @@ -9,10 +14,6 @@ import ( "github.com/diggerhq/digger/backend/utils" "github.com/gin-gonic/gin" "github.com/google/go-github/v61/github" - "log/slog" - "net/http" - "reflect" - "strconv" ) type IssueCommentHook func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error @@ -114,6 +115,26 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) { handlePullRequestEvent(gh, event, d.CiBackendProvider, appId64) }(c.Request.Context()) + case *github.CheckRunEvent: + slog.Info("Processing CheckRunEvent", + "action", event.GetAction(), + "checkRunID", event.GetCheckRun().GetID(), + ) + + // Only care about button clicks: + if event.GetAction() != "requested_action" { + // e.g. "created", "completed", etc. – ignore for now + return + } + + ra := event.GetRequestedAction() + if ra == nil { + slog.Warn("requested_action is nil in CheckRunEvent") + return + } + + identifier := ra.Identifier + slog.Info("Processing CheckRun requested_action", "identifier", identifier) default: slog.Debug("Unhandled event type", "eventType", reflect.TypeOf(event)) } diff --git a/backend/controllers/github_comment.go b/backend/controllers/github_comment.go index 4057dc13d..a74b94baf 100644 --- a/backend/controllers/github_comment.go +++ b/backend/controllers/github_comment.go @@ -574,6 +574,16 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu "batchId", batchId, ) + batch, err := models.DB.GetDiggerBatch(batchId) + if err != nil { + slog.Error("Error getting Digger batch", + "batchId", batchId, + "error", err, + ) + commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not retrieve created batch: %v", err)) + return fmt.Errorf("error getting digger batch") + } + if config.CommentRenderMode == digger_config.CommentRenderModeGroupByModule && (*diggerCommand == scheduler.DiggerCommandPlan || *diggerCommand == scheduler.DiggerCommandApply) { @@ -589,15 +599,6 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu return fmt.Errorf("error posting initial comments") } - batch, err := models.DB.GetDiggerBatch(batchId) - if err != nil { - slog.Error("Error getting Digger batch", - "batchId", batchId, - "error", err, - ) - commentReporterManager.UpdateComment(fmt.Sprintf(":x: PostInitialSourceComments error: %v", err)) - return fmt.Errorf("error getting digger batch") - } batch.SourceDetails, err = json.Marshal(sourceDetails) if err != nil { diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index 12547033e..f898b30f5 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -514,6 +514,16 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR "batchId", batchId, ) + batch, err := models.DB.GetDiggerBatch(batchId) + if err != nil { + slog.Error("Error getting Digger batch", + "batchId", batchId, + "error", err, + ) + commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not retrieve created batch: %v", err)) + return fmt.Errorf("error getting digger batch") + } + if config.CommentRenderMode == digger_config.CommentRenderModeGroupByModule { slog.Info("Using GroupByModule render mode for comments", "prNumber", prNumber) @@ -527,16 +537,6 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR return fmt.Errorf("error posting initial comments") } - batch, err := models.DB.GetDiggerBatch(batchId) - if err != nil { - slog.Error("Error getting Digger batch", - "batchId", batchId, - "error", err, - ) - commentReporterManager.UpdateComment(fmt.Sprintf(":x: PostInitialSourceComments error: %v", err)) - return fmt.Errorf("error getting digger batch") - } - batch.SourceDetails, err = json.Marshal(sourceDetails) if err != nil { slog.Error("Error marshalling source details", diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index e25c162d0..6b3ed7390 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -261,10 +261,32 @@ func UpdateCheckRunForBatch(gh utils.GithubClientProvider, batch *models.DiggerB } if isPlanBatch { - ghPrService.UpdateCheckRun(*batch.CheckRunId, serializedBatch.ToCheckRunStatus(), serializedBatch.ToCheckRunConclusion(), "Plans Summary", summary, message) + status := serializedBatch.ToCheckRunStatus() + conclusion := serializedBatch.ToCheckRunConclusion() + title := "Plans Summary" + opts := github.GithubCheckRunUpdateOptions{ + &status, + conclusion, + &title, + &summary, + &message, + utils.GetActionsForBatch(batch), + } + ghPrService.UpdateCheckRun(*batch.CheckRunId, opts) } else { if disableDiggerApplyStatusCheck == false { - ghPrService.UpdateCheckRun(*batch.CheckRunId, serializedBatch.ToCheckRunStatus(), serializedBatch.ToCheckRunConclusion(), "Apply Summary", summary, message) + status := serializedBatch.ToCheckRunStatus() + conclusion := serializedBatch.ToCheckRunConclusion() + title := "Apply Summary" + opts := github.GithubCheckRunUpdateOptions{ + &status, + conclusion, + &title, + &summary, + &message, + utils.GetActionsForBatch(batch), + } + ghPrService.UpdateCheckRun(*batch.CheckRunId, opts) } } return nil @@ -373,13 +395,29 @@ func UpdateCheckRunForJob(gh utils.GithubClientProvider, job *models.DiggerJob) slog.Debug("Updating PR status for job", "jobId", job.DiggerJobID, "status", status, "conclusion", conclusion) if isPlan { title := fmt.Sprintf("%v to create %v to update %v to delete", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted) - _, err = ghService.UpdateCheckRun(*job.CheckRunId, status, conclusion, title, summary, text) + opts := github.GithubCheckRunUpdateOptions{ + Status: &status, + Conclusion: &conclusion, + Title: &title, + Summary: &summary, + Text: &text, + Actions: utils.GetActionsForJob(job), + } + _, err = ghService.UpdateCheckRun(*job.CheckRunId, opts) if err != nil { slog.Error("Error updating PR status for job", "error", err) } } else { title := fmt.Sprintf("%v created %v updated %v deleted", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted) - _, err = ghService.UpdateCheckRun(*job.CheckRunId, status, conclusion, title, summary, text) + opts := github.GithubCheckRunUpdateOptions{ + Status: &status, + Conclusion: &conclusion, + Title: &title, + Summary: &summary, + Text: &text, + Actions: utils.GetActionsForJob(job), + } + _, err = ghService.UpdateCheckRun(*job.CheckRunId, opts) slog.Error("Error updating PR status for job", "error", err) } return nil diff --git a/backend/migrations/20251120020911.sql b/backend/migrations/20251120020911.sql new file mode 100644 index 000000000..2f73207bc --- /dev/null +++ b/backend/migrations/20251120020911.sql @@ -0,0 +1,2 @@ +-- Modify "digger_batches" table +ALTER TABLE "public"."digger_batches" ADD COLUMN "digger_batch_id" text NULL; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index 095d82242..67aa361ea 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:KHwly+q9MAIOsJSqgLlXD/pJHaVU/ObSzdiaeby9+Tg= +h1:dBuMCfDR69TtAIcc5I5aABz/YqSW6HDVOlmnTyLmKl0= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -71,3 +71,4 @@ h1:KHwly+q9MAIOsJSqgLlXD/pJHaVU/ObSzdiaeby9+Tg= 20251114230419.sql h1:/WA7vp7SKgdfe3KHS65nbwE4RUyUmlUOxuQ8tZZ/FQI= 20251118022613.sql h1:nSMv/SJ6gUdKjWWVJJhgPTN18SQrhkkknBJGnx8lhKc= 20251119004103.sql h1:zdyEn54C6mY5iKZ86LQWhOi13sSA2EMriE1lQ9wGi6w= +20251120020911.sql h1:JaybKP/PHLE3qt5+jA9k0sGFAMPl62T91SSMOC3W5Ow= diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 8e3576cf7..9b9d2d430 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -38,6 +38,7 @@ const DiggerVCSBitbucket DiggerVCSType = "bitbucket" type DiggerBatch struct { gorm.Model ID uuid.UUID `gorm:"primary_key"` + DiggerBatchID string `gorm:"size:20,index:idx_digger_batch_id"` // shorter version of the ID to be able to use in check run Layer uint VCS DiggerVCSType PrNumber int @@ -218,7 +219,6 @@ func GetCommitStatusForJob(job *DiggerJob) (string, error) { return "", fmt.Errorf("unknown job status: %v", job.Status) } - func GetCheckRunStatusForJob(job *DiggerJob) (string, error) { switch job.Status { case orchestrator_scheduler.DiggerJobStarted: diff --git a/backend/models/storage.go b/backend/models/storage.go index e0f50d087..9f40b5cb9 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -868,8 +868,10 @@ func (db *Database) GetDiggerBatch(batchId *uuid.UUID) (*DiggerBatch, error) { func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool, coverAllImpactedProjects bool, VCSConnectionId *uint, commitSha string, checkRunId *string, checkRunUrl *string) (*DiggerBatch, error) { uid := uuid.New() + diggerBatchId := uniuri.NewLen(20) batch := &DiggerBatch{ ID: uid, + DiggerBatchID: diggerBatchId, VCS: vcsType, VCSConnectionId: VCSConnectionId, GithubInstallationId: githubInstallationId, diff --git a/backend/utils/github.go b/backend/utils/github.go index 768f75217..e078c0c95 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -264,7 +264,8 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc "prNumber", prNumber, "project", job.ProjectName, ) - cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/plan", "in_progress", "", "Waiting for plan..." , "", "Plan result will appear here", commitSha) + var actions []*github.CheckRunAction + cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/plan", "in_progress", "", "Waiting for plan...", "", "Plan result will appear here", commitSha, actions) jobCheckRunIds[job.ProjectName] = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), Url: *cr.HTMLURL, @@ -275,7 +276,7 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc "prNumber", prNumber, "project", job.ProjectName, ) - cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/apply", "in_progress", "", "Waiting for apply..." , "", "Apply result will appear here", commitSha) + cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/apply", "in_progress", "", "Waiting for apply...", "", "Apply result will appear here", commitSha, nil) jobCheckRunIds[job.ProjectName] = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), Url: *cr.URL, @@ -300,14 +301,14 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc var cr *github.CheckRun if scheduler.IsPlanJobs(jobs) { slog.Debug("Setting aggregate plan status", "prNumber", prNumber) - cr, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "Pending start..." , "", jobsSummaryTable, commitSha) + cr, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "Pending start...", "", jobsSummaryTable, commitSha, nil) batchCheckRunId = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), Url: *cr.HTMLURL, } } else { slog.Debug("Setting aggregate apply status", "prNumber", prNumber) - cr, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "Pending start..." , "", jobsSummaryTable, commitSha) + cr, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "Pending start...", "", jobsSummaryTable, commitSha, nil) batchCheckRunId = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), Url: *cr.HTMLURL, @@ -322,13 +323,13 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc } } else { slog.Debug("Setting success status for empty job list", "prNumber", prNumber) - _, err := ghService.CreateCheckRun("digger/plan", "completed", "success", "No impacted projects" , "Check your configuration and files changed if this is unexpected", "digger/plan", commitSha) + _, err := ghService.CreateCheckRun("digger/plan", "completed", "success", "No impacted projects", "Check your configuration and files changed if this is unexpected", "digger/plan", commitSha, nil) if err != nil { slog.Error("Failed to set success plan status", "prNumber", prNumber, "error", err) return nil, nil, fmt.Errorf("error setting pr status: %v", err) } - _, err = ghService.CreateCheckRun("digger/apply", "completed", "success", "No impacted projects" , "Check your configuration and files changed if this is unexpected", "digger/apply", commitSha) + _, err = ghService.CreateCheckRun("digger/apply", "completed", "success", "No impacted projects", "Check your configuration and files changed if this is unexpected", "digger/apply", commitSha, nil) if err != nil { slog.Error("Failed to set success apply status", "prNumber", prNumber, "error", err) return nil, nil, fmt.Errorf("error setting pr status: %v", err) @@ -339,6 +340,30 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc return &batchCheckRunId, jobCheckRunIds, nil } +func GetActionsForBatch(batch *models.DiggerBatch) []*github.CheckRunAction { + batchActions := make([]*github.CheckRunAction, 0) + if batch.Status == scheduler.BatchJobSucceeded { + batchActions = append(batchActions, &github.CheckRunAction{ + Label: "Apply all", // max 20 chars + Description: "Apply all jobs", // max 40 chars + Identifier: batch.DiggerBatchID, // max 20 chars + }) + } + return batchActions +} + +func GetActionsForJob(job *models.DiggerJob) []*github.CheckRunAction { + batchActions := make([]*github.CheckRunAction, 0) + if job.Status == scheduler.DiggerJobSucceeded { + batchActions = append(batchActions, &github.CheckRunAction{ + Label: "Apply job", // max 20 chars + Description: "Apply this job", // max 40 chars + Identifier: job.DiggerJobID, // max 20 chars + }) + } + return batchActions +} + func GetGithubHostname() string { githubHostname := os.Getenv("DIGGER_GITHUB_HOSTNAME") if githubHostname == "" { diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index ff5cbd094..f86583356 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -309,7 +309,7 @@ func (svc GithubService) SetStatus(prNumber int, status string, statusContext st } // modern check runs for github (not the commit status) -func (svc GithubService) CreateCheckRun(name string, status string, conclusion string, title string, summary string, text string, headSHA string) (*github.CheckRun, error) { +func (svc GithubService) CreateCheckRun(name string, status string, conclusion string, title string, summary string, text string, headSHA string, actions []*github.CheckRunAction) (*github.CheckRun, error) { client := svc.Client owner := svc.Owner repoName := svc.RepoName @@ -328,12 +328,32 @@ func (svc GithubService) CreateCheckRun(name string, status string, conclusion s opts.Conclusion = github.String(conclusion) } + if actions != nil { + opts.Actions = actions + } + ctx := context.Background() checkRun, _, err := client.Checks.CreateCheckRun(ctx, owner, repoName, opts) return checkRun, err } -func (svc GithubService) UpdateCheckRun(checkRunId string, status string, conclusion string, title string, summary string, text string) (*github.CheckRun, error) { +type GithubCheckRunUpdateOptions struct { + Status *string + Conclusion *string + Title *string + Summary *string + Text *string + Actions []*github.CheckRunAction +} + +func (svc GithubService) UpdateCheckRun(checkRunId string, options GithubCheckRunUpdateOptions) (*github.CheckRun, error) { + status := options.Status + conclusion := options.Conclusion + title := options.Title + summary := options.Summary + text := options.Text + actions := options.Actions + slog.Debug("Updating check run", "checkRunId", checkRunId, "status", status, @@ -341,6 +361,7 @@ func (svc GithubService) UpdateCheckRun(checkRunId string, status string, conclu "title", title, "summary", summary, "text", text, + "actions", actions, ) client := svc.Client owner := svc.Owner @@ -376,33 +397,49 @@ func (svc GithubService) UpdateCheckRun(checkRunId string, status string, conclu } } + newActions := []*github.CheckRunAction{} + if actions != nil { + newActions = actions + } + // Update with new values (only update if provided and non-empty) - if title != "" { - output.Title = github.String(title) + if title != nil { + output.Title = github.String(*title) } else if existingCheckRun.Output != nil && existingCheckRun.Output.Title != nil { output.Title = existingCheckRun.Output.Title } - if summary != "" { - output.Summary = github.String(summary) + if summary != nil { + output.Summary = github.String(*summary) } else if existingCheckRun.Output != nil && existingCheckRun.Output.Summary != nil { output.Summary = existingCheckRun.Output.Summary } - if text != "" { - output.Text = github.String(text) + if text != nil { + output.Text = github.String(*text) } else if existingCheckRun.Output != nil && existingCheckRun.Output.Text != nil { output.Text = existingCheckRun.Output.Text } + var newStatus *string = nil + if status != nil { + newStatus = status + } else { + newStatus = existingCheckRun.Status + } + opts := github.UpdateCheckRunOptions{ Name: *existingCheckRun.Name, - Status: github.String(status), Output: output, + Actions: newActions, } - if conclusion != "" { - opts.Conclusion = github.String(conclusion) + if newStatus != nil { + opts.Status = github.String(*newStatus) + } + + if conclusion != nil { + opts.Conclusion = github.String(*conclusion) } checkRun, _, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts) @@ -413,20 +450,6 @@ func (svc GithubService) UpdateCheckRun(checkRunId string, status string, conclu return checkRun, err } - // Log both input and output IDs for verification - if checkRun != nil && checkRun.ID != nil { - returnedIdStr := strconv.FormatInt(*checkRun.ID, 10) - slog.Debug("Check run updated", - "inputCheckRunId", checkRunId, - "returnedCheckRunId", returnedIdStr, - "externalId", checkRun.ExternalID, - "status", status, - "conclusion", conclusion) - } else { - slog.Warn("Check run updated but returned nil ID", - "inputCheckRunId", checkRunId) - } - return checkRun, err } diff --git a/libs/scheduler/models.go b/libs/scheduler/models.go index 33185ec3d..041bb1890 100644 --- a/libs/scheduler/models.go +++ b/libs/scheduler/models.go @@ -176,18 +176,21 @@ func (b *SerializedBatch) ToCheckRunStatus() string { } } -func (b *SerializedBatch) ToCheckRunConclusion() string { +func (b *SerializedBatch) ToCheckRunConclusion() *string { switch b.Status { case BatchJobCreated: - return "" + return nil case BatchJobInvalidated: - return "cancelled" + res := "cancelled" + return &res case BatchJobFailed: - return "failure" + res := "failure" + return &res case BatchJobSucceeded: - return "success" + res := "success" + return &res default: - return "" + return nil } } From bdea49b23c1e118806fdab2487644d385bab8f95 Mon Sep 17 00:00:00 2001 From: motatoes Date: Wed, 19 Nov 2025 21:39:06 -0800 Subject: [PATCH 07/12] checkpoint --- backend/controllers/github.go | 5 ++ backend/controllers/github_pull_request.go | 2 +- backend/controllers/gitihub_check_run.go | 100 +++++++++++++++++++++ backend/models/storage.go | 15 +++- backend/utils/github.go | 8 +- 5 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 backend/controllers/gitihub_check_run.go diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 67295ffe6..b2db10a40 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -134,6 +134,11 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) { } identifier := ra.Identifier + // run it as a goroutine to avoid timeouts + go func(ctx context.Context) { + defer logging.InheritRequestLogger(ctx)() + handleCheckRunActionEvent(gh, event, d.CiBackendProvider, appId64) + }(c.Request.Context()) slog.Info("Processing CheckRun requested_action", "identifier", identifier) default: slog.Debug("Unhandled event type", "eventType", reflect.TypeOf(event)) diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index f898b30f5..fc137b4e5 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -523,7 +523,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not retrieve created batch: %v", err)) return fmt.Errorf("error getting digger batch") } - + if config.CommentRenderMode == digger_config.CommentRenderModeGroupByModule { slog.Info("Using GroupByModule render mode for comments", "prNumber", prNumber) diff --git a/backend/controllers/gitihub_check_run.go b/backend/controllers/gitihub_check_run.go new file mode 100644 index 000000000..5404ae0c9 --- /dev/null +++ b/backend/controllers/gitihub_check_run.go @@ -0,0 +1,100 @@ +package controllers + +import ( + "fmt" + "log/slog" + "runtime/debug" + + "github.com/diggerhq/digger/backend/ci_backends" + "github.com/diggerhq/digger/backend/utils" + "github.com/google/go-github/v61/github" +) + +func handleCheckRunActionEvent(gh utils.GithubClientProvider, identifier string, payload *github.CheckRunEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error { + defer func() { + if r := recover(); r != nil { + stack := string(debug.Stack()) + slog.Error("Recovered from panic in handlePullRequestEvent", "error", r, slog.Group("stack")) + fmt.Printf("Stack trace:\n%s\n", stack) + } + }() + + //repoFullName := *payload.Repo.FullName + //repoName := *payload.Repo.Name + //repoOwner := *payload.Repo.Owner.Login + //cloneUrl := *payload.Repo.CloneURL + // + //checkRunBatch, err := models.DB.GetDiggerBatchFromId(identifier) + //if err != nil { + // slog.Error("Failed to find batch from identifier %v, err: %v", identifier, err) + // return fmt.Errorf("Failed to find batch from identifier %v, err: %v", identifier, err) + //} + // + //installationId := checkRunBatch.GithubInstallationId + //prNumber := checkRunBatch.PrNumber + // + //link, err := models.DB.GetGithubAppInstallationLink(installationId) + //if err != nil { + // slog.Error("Error getting GitHub app installation link", + // "installationId", installationId, + // "error", err, + // ) + // return fmt.Errorf("error getting github app link") + //} + //if link == nil { + // slog.Error("GitHub app installation link not found", + // "installationId", installationId, + // "prNumber", prNumber, + // ) + // return fmt.Errorf("GitHub App installation not found for installation ID %d. Please ensure the GitHub App is properly installed on the repository and the installation process completed successfully", installationId) + //} + //orgId := link.OrganisationId + //prLabelsStr := make([]string, 0) + + + //diggerYmlStr, ghService, config, projectsGraph, prSourceBranch, commitSha, _, err := getDiggerConfigForPR(gh, orgId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneUrl, prNumber) + //if err != nil { + // slog.Error("Error getting Digger config for PR", + // "issueNumber", prNumber, + // "repoFullName", repoFullName, + // "error", err, + // ) + // return fmt.Errorf("error getting digger config") + //} + + //batchId, _, err := utils.ConvertJobsToDiggerJobs( + // scheduler.DiggerCommandApply, + // "github", + // orgId, + // impactedProjectsJobMap, + // impactedProjectsMap, + // projectsGraph, + // installationId, + // *prSourceBranch, + // prNumber, + // repoOwner, + // repoName, + // repoFullName, + // *commitSha, + // reporterCommentId, + // diggerYmlStr, + // 0, + // "", + // config.ReportTerraformOutputs, + // coverAllImpactedProjects, + // nil, + // batchCheckRunData, + // jobCheckRunDataMap, + //) + // + //if err != nil { + // slog.Error("Error converting jobs to Digger jobs", + // "issueNumber", prNumber, + // "error", err, + // ) + // commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) + // return fmt.Errorf("error converting jobs") + //} + + return nil +} \ No newline at end of file diff --git a/backend/models/storage.go b/backend/models/storage.go index 9f40b5cb9..4235bd05b 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -866,9 +866,20 @@ func (db *Database) GetDiggerBatch(batchId *uuid.UUID) (*DiggerBatch, error) { return batch, nil } +func (db *Database) GetDiggerBatchFromId(diggerBatchId string) (*DiggerBatch, error) { + batch := &DiggerBatch{} + result := db.GormDB.Where("digger_batch_id=? ", diggerBatchId).Find(batch) + if result.Error != nil { + if !errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, result.Error + } + } + return batch, nil +} + func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool, coverAllImpactedProjects bool, VCSConnectionId *uint, commitSha string, checkRunId *string, checkRunUrl *string) (*DiggerBatch, error) { uid := uuid.New() - diggerBatchId := uniuri.NewLen(20) + diggerBatchId := uniuri.NewLen(7) batch := &DiggerBatch{ ID: uid, DiggerBatchID: diggerBatchId, @@ -953,7 +964,7 @@ func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, wor if serializedJob == nil || len(serializedJob) == 0 { return nil, fmt.Errorf("serializedJob can't be empty") } - jobId := uniuri.New() + jobId := uniuri.NewLen(10) batchIdStr := batchId.String() summary := &DiggerJobSummary{} diff --git a/backend/utils/github.go b/backend/utils/github.go index e078c0c95..cbe7dc30b 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -340,13 +340,17 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc return &batchCheckRunId, jobCheckRunIds, nil } +type CheckedRunActionIdentifier string +const CheckedRunActionBatchApply CheckedRunActionIdentifier = "abatch" +const CheckedRunActionJobApply CheckedRunActionIdentifier = "ajob" + func GetActionsForBatch(batch *models.DiggerBatch) []*github.CheckRunAction { batchActions := make([]*github.CheckRunAction, 0) if batch.Status == scheduler.BatchJobSucceeded { batchActions = append(batchActions, &github.CheckRunAction{ Label: "Apply all", // max 20 chars Description: "Apply all jobs", // max 40 chars - Identifier: batch.DiggerBatchID, // max 20 chars + Identifier: fmt.Sprintf(batch.DiggerBatchID), // max 20 chars }) } return batchActions @@ -358,7 +362,7 @@ func GetActionsForJob(job *models.DiggerJob) []*github.CheckRunAction { batchActions = append(batchActions, &github.CheckRunAction{ Label: "Apply job", // max 20 chars Description: "Apply this job", // max 40 chars - Identifier: job.DiggerJobID, // max 20 chars + Identifier: fmt.Sprintf("ajob:%v", job.DiggerJobID), // max 20 chars }) } return batchActions From fa9530b377c1e18a4baef660cb4148fe2e609211 Mon Sep 17 00:00:00 2001 From: motatoes Date: Wed, 19 Nov 2025 22:02:02 -0800 Subject: [PATCH 08/12] checkpoint --- backend/controllers/github_comment.go | 8 +++++- backend/controllers/github_pull_request.go | 29 ++++------------------ backend/controllers/github_test.go | 8 +++--- backend/migrations/20251120060106.sql | 2 ++ backend/migrations/atlas.sum | 3 ++- backend/models/scheduler.go | 1 + backend/models/scheduler_test.go | 10 ++++---- backend/models/storage.go | 3 ++- backend/models/storage_test.go | 2 +- backend/utils/graphs.go | 6 ++--- ee/backend/controllers/gitlab.go | 4 +-- ee/backend/hooks/github.go | 2 +- libs/digger_config/config.go | 1 + libs/digger_config/converters.go | 2 ++ libs/digger_config/yaml.go | 11 ++++++++ 15 files changed, 49 insertions(+), 43 deletions(-) create mode 100644 backend/migrations/20251120060106.sql diff --git a/backend/controllers/github_comment.go b/backend/controllers/github_comment.go index a74b94baf..51e722f14 100644 --- a/backend/controllers/github_comment.go +++ b/backend/controllers/github_comment.go @@ -553,13 +553,19 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu slog.Debug("Created AI summary comment", "commentId", aiSummaryCommentId) } + + reporterType := "lazy" + if config.Reporting.CommentsEnabled == false { + reporterType = "noop" + } + slog.Info("Converting jobs to Digger jobs", "issueNumber", issueNumber, "command", *diggerCommand, "jobCount", len(impactedProjectsJobMap), ) - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *prSourceBranch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, batchCheckRunData, jobCheckRunDataMap) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, reporterType, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *prSourceBranch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, batchCheckRunData, jobCheckRunDataMap) if err != nil { slog.Error("Error converting jobs to Digger jobs", "issueNumber", issueNumber, diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index fc137b4e5..2143db42e 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -466,6 +466,10 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR slog.Debug("Created AI summary comment", "commentId", aiSummaryCommentId) } + reporterType := "lazy" + if config.Reporting.CommentsEnabled == false { + reporterType = "noop" + } slog.Info("Converting jobs to Digger jobs", "prNumber", prNumber, "command", *diggerCommand, @@ -475,30 +479,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR if config.RespectLayers { } - batchId, _, err := utils.ConvertJobsToDiggerJobs( - *diggerCommand, - models.DiggerVCSGithub, - organisationId, - impactedJobsMap, - impactedProjectsMap, - projectsGraph, - installationId, - branch, - prNumber, - repoOwner, - repoName, - repoFullName, - commitSha, - commentId, - diggerYmlStr, - 0, - aiSummaryCommentId, - config.ReportTerraformOutputs, - coverAllImpactedProjects, - nil, - batchCheckRunData, - jobsCheckRunIdsMap, - ) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, reporterType, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, batchCheckRunData, jobsCheckRunIdsMap) if err != nil { slog.Error("Error converting jobs to Digger jobs", "prNumber", prNumber, diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go index f4c8cabc1..91026815d 100644 --- a/backend/controllers/github_test.go +++ b/backend/controllers/github_test.go @@ -724,7 +724,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) { graph, err := configuration.CreateProjectDependencyGraph(projects) assert.NoError(t, err) - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, true, nil, nil, nil) + _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 1, len(result)) @@ -754,7 +754,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) + _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -788,7 +788,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) + _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -834,7 +834,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) { projectMap["555"] = project5 projectMap["666"] = project6 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) + _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 6, len(result)) diff --git a/backend/migrations/20251120060106.sql b/backend/migrations/20251120060106.sql new file mode 100644 index 000000000..dcad2230f --- /dev/null +++ b/backend/migrations/20251120060106.sql @@ -0,0 +1,2 @@ +-- Modify "digger_jobs" table +ALTER TABLE "public"."digger_jobs" ADD COLUMN "reporter_type" text NULL DEFAULT 'lazy'; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index 67aa361ea..803d792ad 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:dBuMCfDR69TtAIcc5I5aABz/YqSW6HDVOlmnTyLmKl0= +h1:Yoa3j5DOMmrwiYi3e7uIy5Y32t7e4jmjUxerP0qXEGA= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -72,3 +72,4 @@ h1:dBuMCfDR69TtAIcc5I5aABz/YqSW6HDVOlmnTyLmKl0= 20251118022613.sql h1:nSMv/SJ6gUdKjWWVJJhgPTN18SQrhkkknBJGnx8lhKc= 20251119004103.sql h1:zdyEn54C6mY5iKZ86LQWhOi13sSA2EMriE1lQ9wGi6w= 20251120020911.sql h1:JaybKP/PHLE3qt5+jA9k0sGFAMPl62T91SSMOC3W5Ow= +20251120060106.sql h1:MK5LjwWUr3nszLIzSJJBAy7d8Y2PvpDRV8qmTTnFfIM= diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 9b9d2d430..77cadb7ea 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -92,6 +92,7 @@ type DiggerJob struct { WorkflowFile string WorkflowRunUrl *string StatusUpdatedAt time.Time + ReporterType string `gorm:"default:'lazy'"` // temporary, to be replaced by SerializedReporterSpec } type DiggerJobSummary struct { diff --git a/backend/models/scheduler_test.go b/backend/models/scheduler_test.go index 9cb4675f4..892a95e85 100644 --- a/backend/models/scheduler_test.go +++ b/backend/models/scheduler_test.go @@ -75,7 +75,7 @@ func TestCreateDiggerJob(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") assert.NoError(t, err) assert.NotNil(t, job) @@ -87,7 +87,7 @@ func TestCreateSingleJob(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") assert.NoError(t, err) assert.NotNil(t, job) @@ -99,20 +99,20 @@ func TestFindDiggerJobsByParentJobId(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") parentJobId := job.DiggerJobID assert.NoError(t, err) assert.NotNil(t, job) assert.NotZero(t, job.ID) - job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) + job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") assert.NoError(t, err) assert.NotNil(t, job) assert.NotZero(t, job.ID) err = database.CreateDiggerJobParentLink(parentJobId, job.DiggerJobID) assert.Nil(t, err) - job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil) + job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") assert.NoError(t, err) assert.NotNil(t, job) err = database.CreateDiggerJobParentLink(parentJobId, job.DiggerJobID) diff --git a/backend/models/storage.go b/backend/models/storage.go index 4235bd05b..da7fcb7b0 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -960,7 +960,7 @@ func (db *Database) UpdateBatchStatus(batch *DiggerBatch) error { return nil } -func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, workflowFile string, checkRunId *string, checkRunUrl *string) (*DiggerJob, error) { +func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, workflowFile string, checkRunId *string, checkRunUrl *string, reporterType string) (*DiggerJob, error) { if serializedJob == nil || len(serializedJob) == 0 { return nil, fmt.Errorf("serializedJob can't be empty") } @@ -984,6 +984,7 @@ func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, wor DiggerJobSummary: *summary, WorkflowRunUrl: &workflowUrl, WorkflowFile: workflowFile, + ReporterType: reporterType, } result = db.GormDB.Save(job) if result.Error != nil { diff --git a/backend/models/storage_test.go b/backend/models/storage_test.go index 0b9f847af..0e96e3fc5 100644 --- a/backend/models/storage_test.go +++ b/backend/models/storage_test.go @@ -146,7 +146,7 @@ func TestGetDiggerJobsForBatchPreloadsSummary(t *testing.T) { batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false, true, nil, "", nil, nil) assert.NoError(t, err) - job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml", nil, nil) + job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml", nil, nil, "lazy") assert.NoError(t, err) job, err = DB.UpdateDiggerJobSummary(job.DiggerJobID, resourcesCreated, resourcesUpdated, resourcesDeleted) diff --git a/backend/utils/graphs.go b/backend/utils/graphs.go index 40a8f6429..e933fa509 100644 --- a/backend/utils/graphs.go +++ b/backend/utils/graphs.go @@ -15,7 +15,7 @@ import ( ) // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value -func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool, VCSConnectionId *uint, batchCheckRunData *CheckRunData, jobsCheckRunIdsMap map[string]CheckRunData) (*uuid.UUID, map[string]*models.DiggerJob, error) { +func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, jobReporterType string, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool, VCSConnectionId *uint, batchCheckRunData *CheckRunData, jobsCheckRunIdsMap map[string]CheckRunData) (*uuid.UUID, map[string]*models.DiggerJob, error) { slog.Info("Converting jobs to Digger jobs", "jobType", jobType, "vcsType", vcsType, @@ -109,7 +109,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig } if predecessorMap[value] == nil || len(predecessorMap[value]) == 0 { slog.Debug("Processing node with no parents", "projectName", value) - parentJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl) + parentJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl, jobReporterType) if err != nil { slog.Error("Failed to create job", "projectName", value, @@ -146,7 +146,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig parent := edge.Source parentDiggerJob := result[parent] - childJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl) + childJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl, jobReporterType) if err != nil { slog.Error("Failed to create child job", "projectName", value, diff --git a/ee/backend/controllers/gitlab.go b/ee/backend/controllers/gitlab.go index 29e71dc5f..d7219b90b 100644 --- a/ee/backend/controllers/gitlab.go +++ b/ee/backend/controllers/gitlab.go @@ -288,7 +288,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -488,7 +488,7 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/ee/backend/hooks/github.go b/ee/backend/hooks/github.go index ef8f6114e..3200dee5d 100644 --- a/ee/backend/hooks/github.go +++ b/ee/backend/hooks/github.go @@ -152,7 +152,7 @@ var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.Git utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false, coverAllImpactedProjects, nil, nil, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/libs/digger_config/config.go b/libs/digger_config/config.go index 9e33ea010..7f06c2f7d 100644 --- a/libs/digger_config/config.go +++ b/libs/digger_config/config.go @@ -34,6 +34,7 @@ type DiggerConfig struct { type ReporterConfig struct { AiSummary bool + CommentsEnabled bool } type DependencyConfiguration struct { diff --git a/libs/digger_config/converters.go b/libs/digger_config/converters.go index 3d3b58a4b..20a749190 100644 --- a/libs/digger_config/converters.go +++ b/libs/digger_config/converters.go @@ -194,11 +194,13 @@ func copyReporterConfig(r *ReportingConfigYaml) ReporterConfig { if r == nil { return ReporterConfig{ AiSummary: false, + CommentsEnabled: true, } } return ReporterConfig{ AiSummary: r.AiSummary, + CommentsEnabled: r.CommentsEnabled, } } diff --git a/libs/digger_config/yaml.go b/libs/digger_config/yaml.go index 914dd2478..83bfcf98c 100644 --- a/libs/digger_config/yaml.go +++ b/libs/digger_config/yaml.go @@ -31,6 +31,7 @@ type DiggerConfigYaml struct { type ReportingConfigYaml struct { AiSummary bool `yaml:"ai_summary"` + CommentsEnabled bool `yaml:"comments_enabled"` } type DependencyConfigurationYaml struct { @@ -191,6 +192,16 @@ type TerragruntParsingConfig struct { DependsOnOrdering *bool `yaml:"dependsOnOrdering,omitempty"` } +func (c *ReportingConfigYaml) UnmarshalYAML(unmarshal func(any) error) error { + // set defaults + c.AiSummary = false + c.CommentsEnabled = true + + // overlay YAML values + type plain ReportingConfigYaml + return unmarshal((*plain)(c)) +} + func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { type rawProject ProjectYaml raw := rawProject{ From 105eebb11af2b413f374d3a4119589ba3cf0b4e3 Mon Sep 17 00:00:00 2001 From: motatoes Date: Wed, 19 Nov 2025 22:10:31 -0800 Subject: [PATCH 09/12] checkpoint --- ee/backend/controllers/bitbucket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/backend/controllers/bitbucket.go b/ee/backend/controllers/bitbucket.go index 3f6da11db..f3320219d 100644 --- a/ee/backend/controllers/bitbucket.go +++ b/ee/backend/controllers/bitbucket.go @@ -305,7 +305,7 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false, true, vcsConnectionId, nil, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false, true, vcsConnectionId, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) From e44fb430428cb151241dddee65a604281b462d0c Mon Sep 17 00:00:00 2001 From: motatoes Date: Wed, 19 Nov 2025 22:19:39 -0800 Subject: [PATCH 10/12] check --- backend/controllers/github.go | 2 +- backend/services/spec.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index b2db10a40..6accb0c34 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -137,7 +137,7 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) { // run it as a goroutine to avoid timeouts go func(ctx context.Context) { defer logging.InheritRequestLogger(ctx)() - handleCheckRunActionEvent(gh, event, d.CiBackendProvider, appId64) + handleCheckRunActionEvent(gh, identifier, event, d.CiBackendProvider, appId64) }(c.Request.Context()) slog.Info("Processing CheckRun requested_action", "identifier", identifier) default: diff --git a/backend/services/spec.go b/backend/services/spec.go index fbac384ac..caa48faff 100644 --- a/backend/services/spec.go +++ b/backend/services/spec.go @@ -189,7 +189,7 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { Job: jobSpec, Reporter: spec.ReporterSpec{ ReportingStrategy: "comments_per_run", - ReporterType: "lazy", + ReporterType: job.ReporterType, ReportTerraformOutput: batch.ReportTerraformOutputs, }, Lock: spec.LockSpec{ From 85876ef5241b41cebb0a807937e0467cd5e09d16 Mon Sep 17 00:00:00 2001 From: motatoes Date: Wed, 19 Nov 2025 22:27:25 -0800 Subject: [PATCH 11/12] fix build --- backend/utils/github.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/utils/github.go b/backend/utils/github.go index cbe7dc30b..a0d1a4faa 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -350,7 +350,7 @@ func GetActionsForBatch(batch *models.DiggerBatch) []*github.CheckRunAction { batchActions = append(batchActions, &github.CheckRunAction{ Label: "Apply all", // max 20 chars Description: "Apply all jobs", // max 40 chars - Identifier: fmt.Sprintf(batch.DiggerBatchID), // max 20 chars + Identifier: fmt.Sprintf("%v:%v", CheckedRunActionBatchApply, batch.DiggerBatchID), // max 20 chars }) } return batchActions @@ -362,7 +362,7 @@ func GetActionsForJob(job *models.DiggerJob) []*github.CheckRunAction { batchActions = append(batchActions, &github.CheckRunAction{ Label: "Apply job", // max 20 chars Description: "Apply this job", // max 40 chars - Identifier: fmt.Sprintf("ajob:%v", job.DiggerJobID), // max 20 chars + Identifier: fmt.Sprintf("%v:%v", CheckedRunActionJobApply, job.DiggerJobID), // max 20 chars }) } return batchActions From 1b39926196a0a7c6f2522ff6cfd07d118ac92487 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 20 Nov 2025 22:30:30 -0800 Subject: [PATCH 12/12] final checkpoint --- backend/controllers/github_comment.go | 6 +- backend/controllers/github_helpers.go | 21 +- backend/controllers/github_pull_request.go | 8 +- backend/controllers/github_test.go | 12 +- backend/controllers/gitihub_check_run.go | 253 ++++++++++++++------- backend/controllers/projects_helpers.go | 4 +- backend/models/scheduler_test.go | 10 +- backend/models/storage.go | 3 +- backend/models/storage_test.go | 2 +- backend/utils/comment_utils.go | 5 + backend/utils/github.go | 34 ++- backend/utils/graphs.go | 8 +- ee/backend/controllers/bitbucket.go | 2 +- ee/backend/controllers/gitlab.go | 4 +- ee/backend/hooks/github.go | 4 +- 15 files changed, 252 insertions(+), 124 deletions(-) diff --git a/backend/controllers/github_comment.go b/backend/controllers/github_comment.go index 51e722f14..e57f087df 100644 --- a/backend/controllers/github_comment.go +++ b/backend/controllers/github_comment.go @@ -493,7 +493,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu ) // This one is for aggregate reporting //err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) - _, _, err = utils.SetPRCheckForJobs(ghService, issueNumber, jobs, *commitSha) + _, _, err = utils.SetPRCheckForJobs(ghService, issueNumber, jobs, *commitSha, repoName, repoOwner) return nil } @@ -501,7 +501,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu segment.Track(*org, repoOwner, vcsActorID, "github", "issue_digger_comment", map[string]string{"comment": commentBody}) //err = utils.SetPRCommitStatusForJobs(ghService, issueNumber, jobs) - batchCheckRunData, jobCheckRunDataMap, err := utils.SetPRCheckForJobs(ghService, issueNumber, jobs, *commitSha) + batchCheckRunData, jobCheckRunDataMap, err := utils.SetPRCheckForJobs(ghService, issueNumber, jobs, *commitSha, repoName, repoOwner) if err != nil { slog.Error("Error setting status for PR", "issueNumber", issueNumber, @@ -565,7 +565,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu "jobCount", len(impactedProjectsJobMap), ) - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, reporterType, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *prSourceBranch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, batchCheckRunData, jobCheckRunDataMap) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, reporterType, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *prSourceBranch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, &reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, batchCheckRunData, jobCheckRunDataMap) if err != nil { slog.Error("Error converting jobs to Digger jobs", "issueNumber", issueNumber, diff --git a/backend/controllers/github_helpers.go b/backend/controllers/github_helpers.go index 0f3906310..92159d5a8 100644 --- a/backend/controllers/github_helpers.go +++ b/backend/controllers/github_helpers.go @@ -5,6 +5,13 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "github.com/diggerhq/digger/backend/ci_backends" "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/services" @@ -19,12 +26,6 @@ import ( "github.com/google/uuid" "golang.org/x/oauth2" "gorm.io/gorm" - "log/slog" - "net/http" - "os" - "path/filepath" - "slices" - "strings" ) // why this validation is needed: https://roadie.io/blog/avoid-leaking-github-org-data/ @@ -664,9 +665,7 @@ func GetDiggerConfigForBranchWithGracefulHandling(gh utils.GithubClientProvider, } } - diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch( - gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, branch, changedFiles, taConfig, - ) + diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranchOrSha(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, branch, "", changedFiles, taConfig) if err != nil { errMsg := err.Error() @@ -837,7 +836,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels [] return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, changedFiles, nil } -func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, changedFiles []string, taConfig *tac.AtlantisConfig) (string, *github2.GithubService, *digger_config.DiggerConfig, graph.Graph[string, digger_config.Project], error) { +func GetDiggerConfigForBranchOrSha(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, commitSha string, changedFiles []string, taConfig *tac.AtlantisConfig) (string, *github2.GithubService, *digger_config.DiggerConfig, graph.Graph[string, digger_config.Project], error) { slog.Info("Getting Digger config for branch", slog.Group("repository", slog.String("fullName", repoFullName), @@ -863,7 +862,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 var diggerYmlStr string var dependencyGraph graph.Graph[string, digger_config.Project] - err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { + err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, commitSha, *token, "", func(dir string) error { slog.Debug("Reading Digger config from cloned repository", "directory", dir) diggerYmlStr, err = digger_config.ReadDiggerYmlFileContents(dir) diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index 2143db42e..452d3e926 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -200,8 +200,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR // This one is for aggregate reporting commentReporterManager.UpdateComment(":construction_worker: No projects impacted") } - err = utils.SetPRCommitStatusForJobs(ghService, prNumber, jobsForImpactedProjects) - _, _, err = utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha) + _, _, err = utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha, repoName, repoOwner) return nil } @@ -378,7 +377,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR } //err = utils.SetPRCommitStatusForJobs(ghService, prNumber, jobsForImpactedProjects) - batchCheckRunData, jobsCheckRunIdsMap, err := utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha) + batchCheckRunData, jobsCheckRunIdsMap, err := utils.SetPRCheckForJobs(ghService, prNumber, jobsForImpactedProjects, commitSha, repoName, repoOwner) if err != nil { slog.Error("Error setting status for PR", "prNumber", prNumber, @@ -479,7 +478,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR if config.RespectLayers { } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, reporterType, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, batchCheckRunData, jobsCheckRunIdsMap) + + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, reporterType, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, &commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects, nil, batchCheckRunData, jobsCheckRunIdsMap) if err != nil { slog.Error("Error converting jobs to Digger jobs", "prNumber", prNumber, diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go index 91026815d..15955f9ed 100644 --- a/backend/controllers/github_test.go +++ b/backend/controllers/github_test.go @@ -724,7 +724,8 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) { graph, err := configuration.CreateProjectDependencyGraph(projects) assert.NoError(t, err) - _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, true, nil, nil, nil) + var commentId int64 = 123 + _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", &commentId, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 1, len(result)) @@ -754,7 +755,8 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) + var commentId int64 = 123 + _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", &commentId, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -788,7 +790,8 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) + var commentId int64 = 123 + _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", &commentId, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -834,7 +837,8 @@ func TestJobsTreeWithThreeLevels(t *testing.T) { projectMap["555"] = project5 projectMap["666"] = project6 - _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true, nil, nil, nil) + var commentId int64 = 123 + _, result, err := utils.ConvertJobsToDiggerJobs("", "lazy", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", &commentId, "test", 0, "", false, true, nil, nil, nil) assert.NoError(t, err) assert.Equal(t, 6, len(result)) diff --git a/backend/controllers/gitihub_check_run.go b/backend/controllers/gitihub_check_run.go index 5404ae0c9..5becda25e 100644 --- a/backend/controllers/gitihub_check_run.go +++ b/backend/controllers/gitihub_check_run.go @@ -4,10 +4,16 @@ import ( "fmt" "log/slog" "runtime/debug" + "strings" "github.com/diggerhq/digger/backend/ci_backends" + "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/ci/generic" + "github.com/diggerhq/digger/libs/digger_config" + "github.com/diggerhq/digger/libs/scheduler" "github.com/google/go-github/v61/github" + "github.com/samber/lo" ) func handleCheckRunActionEvent(gh utils.GithubClientProvider, identifier string, payload *github.CheckRunEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error { @@ -19,82 +25,177 @@ func handleCheckRunActionEvent(gh utils.GithubClientProvider, identifier string, } }() - //repoFullName := *payload.Repo.FullName - //repoName := *payload.Repo.Name - //repoOwner := *payload.Repo.Owner.Login - //cloneUrl := *payload.Repo.CloneURL - // - //checkRunBatch, err := models.DB.GetDiggerBatchFromId(identifier) - //if err != nil { - // slog.Error("Failed to find batch from identifier %v, err: %v", identifier, err) - // return fmt.Errorf("Failed to find batch from identifier %v, err: %v", identifier, err) - //} - // - //installationId := checkRunBatch.GithubInstallationId - //prNumber := checkRunBatch.PrNumber - // - //link, err := models.DB.GetGithubAppInstallationLink(installationId) - //if err != nil { - // slog.Error("Error getting GitHub app installation link", - // "installationId", installationId, - // "error", err, - // ) - // return fmt.Errorf("error getting github app link") - //} - //if link == nil { - // slog.Error("GitHub app installation link not found", - // "installationId", installationId, - // "prNumber", prNumber, - // ) - // return fmt.Errorf("GitHub App installation not found for installation ID %d. Please ensure the GitHub App is properly installed on the repository and the installation process completed successfully", installationId) - //} - //orgId := link.OrganisationId - //prLabelsStr := make([]string, 0) - - - //diggerYmlStr, ghService, config, projectsGraph, prSourceBranch, commitSha, _, err := getDiggerConfigForPR(gh, orgId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneUrl, prNumber) - //if err != nil { - // slog.Error("Error getting Digger config for PR", - // "issueNumber", prNumber, - // "repoFullName", repoFullName, - // "error", err, - // ) - // return fmt.Errorf("error getting digger config") - //} - - //batchId, _, err := utils.ConvertJobsToDiggerJobs( - // scheduler.DiggerCommandApply, - // "github", - // orgId, - // impactedProjectsJobMap, - // impactedProjectsMap, - // projectsGraph, - // installationId, - // *prSourceBranch, - // prNumber, - // repoOwner, - // repoName, - // repoFullName, - // *commitSha, - // reporterCommentId, - // diggerYmlStr, - // 0, - // "", - // config.ReportTerraformOutputs, - // coverAllImpactedProjects, - // nil, - // batchCheckRunData, - // jobCheckRunDataMap, - //) - // - //if err != nil { - // slog.Error("Error converting jobs to Digger jobs", - // "issueNumber", prNumber, - // "error", err, - // ) - // commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) - // return fmt.Errorf("error converting jobs") - //} + repoFullName := *payload.Repo.FullName + repoName := *payload.Repo.Name + repoOwner := *payload.Repo.Owner.Login + cloneUrl := *payload.Repo.CloneURL + actor := *payload.Sender.Login + var checkRunBatch *models.DiggerBatch + var checkedRunDiggerJobs []models.DiggerJob + + batchCheckApplyAllPrefix := string(utils.CheckedRunActionBatchApply)+":" + if strings.HasPrefix(identifier, batchCheckApplyAllPrefix) { + diggerBatchId := strings.ReplaceAll(identifier, batchCheckApplyAllPrefix, "") + var err error + checkRunBatch, err = models.DB.GetDiggerBatchFromId(diggerBatchId) + if err != nil { + slog.Error("Failed to find batch", "identifier", identifier, "error", err) + return fmt.Errorf("Failed to find batch from identifier %v, err: %v", identifier, err) + } + checkedRunDiggerJobs, err = models.DB.GetDiggerJobsForBatch(checkRunBatch.ID) + if err != nil { + slog.Error("Failed to find jobs for batch", "batchId", checkRunBatch.ID, "error", err) + return fmt.Errorf("Failed to find batch from identifier %v, err: %v", identifier, err) + } + } + + + installationId := checkRunBatch.GithubInstallationId + prNumber := checkRunBatch.PrNumber + commitSha := checkRunBatch.CommitSha + + link, err := models.DB.GetGithubAppInstallationLink(installationId) + if err != nil { + slog.Error("Error getting GitHub app installation link", + "installationId", installationId, + "error", err, + ) + return fmt.Errorf("error getting github app link") + } + if link == nil { + slog.Error("GitHub app installation link not found", + "installationId", installationId, + "prNumber", prNumber, + ) + return fmt.Errorf("GitHub App installation not found for installation ID %d. Please ensure the GitHub App is properly installed on the repository and the installation process completed successfully", installationId) + } + orgId := link.OrganisationId + + + ghService, _, ghServiceErr := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName) + if ghServiceErr != nil { + slog.Error("Error getting GitHub service", + "installationId", installationId, + "repoFullName", repoFullName, + "issueNumber", prNumber, + "error", ghServiceErr, + ) + return fmt.Errorf("error getting ghService to post error comment") + } + + prBranchName, _, _, _, err := ghService.GetBranchName(prNumber) + + + diggerYmlStr, ghService, config, projectsGraph, err := GetDiggerConfigForBranchOrSha(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranchName, commitSha, nil, nil) + if err != nil { + slog.Error("Error getting Digger config for PR", + "issueNumber", prNumber, + "repoFullName", repoFullName, + "error", err, + ) + return fmt.Errorf("error getting digger config") + } + + selectedProjects := lo.Filter(config.Projects, func(diggerYmlProject digger_config.Project, index int) bool { + return lo.ContainsBy(checkedRunDiggerJobs, func(diggerJob models.DiggerJob) bool { + return diggerJob.ProjectName == diggerYmlProject.Name + }) + }) + + jobs, err := generic.CreateJobsForProjects(selectedProjects, "digger apply", "check_run_action", repoFullName, actor, config.Workflows, &prNumber, &commitSha, "", checkRunBatch.BranchName, false) + + + // just use noop since if someone clicks a button he shouldn't see comments on the PR (confusing) + reporterType := "noop" + + + impactedProjectsMap := make(map[string]digger_config.Project) + for _, p := range selectedProjects { + impactedProjectsMap[p.Name] = p + } + + impactedProjectsJobMap := make(map[string]scheduler.Job) + for _, j := range jobs { + impactedProjectsJobMap[j.ProjectName] = j + } + + batchCheckRunData, jobCheckRunDataMap, err := utils.SetPRCheckForJobs(ghService, prNumber, jobs, commitSha, repoName, repoOwner) + if err != nil { + slog.Error("Error setting status for PR", + "prNumber", prNumber, + "error", err, + ) + return fmt.Errorf("error setting status for PR: %v", err) + } + + batchId, _, err := utils.ConvertJobsToDiggerJobs( + scheduler.DiggerCommandApply, + reporterType, + "github", + orgId, + impactedProjectsJobMap, + impactedProjectsMap, + projectsGraph, + installationId, + prBranchName, + prNumber, + repoOwner, + repoName, + repoFullName, + commitSha, + nil, + diggerYmlStr, + 0, + "", + config.ReportTerraformOutputs, + false, + nil, + batchCheckRunData, + jobCheckRunDataMap, + ) + if err != nil { + slog.Error("Error converting jobs to Digger jobs", + "issueNumber", prNumber, + "error", err, + ) + return fmt.Errorf("error converting jobs") + } + + ciBackend, err := ciBackendProvider.GetCiBackend( + ci_backends.CiBackendOptions{ + GithubClientProvider: gh, + GithubInstallationId: installationId, + GithubAppId: appId, + RepoName: repoName, + RepoOwner: repoOwner, + RepoFullName: repoFullName, + }, + ) + if err != nil { + slog.Error("Error getting CI backend", + "prNumber", prNumber, + "repoFullName", repoFullName, + "error", err, + ) + return fmt.Errorf("error fetching ci backed %v", err) + } + + + err = TriggerDiggerJobs(ciBackend, repoFullName, repoOwner, repoName, batchId, prNumber, ghService, gh) + if err != nil { + slog.Error("Error triggering Digger jobs", + "prNumber", prNumber, + "batchId", batchId, + "error", err, + ) + return fmt.Errorf("error triggering Digger Jobs") + } + + slog.Info("Successfully processed issue comment event", + "prNumber", prNumber, + "batchId", batchId, + "repoFullName", repoFullName, + ) return nil } \ No newline at end of file diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index 6b3ed7390..54d8e02cf 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -67,7 +67,7 @@ func GenerateChecksSummaryForBatch( batch *models.DiggerBatch) (string, error) { summary := "" if aiSummary != "FOUR_OH_FOUR" { - summary = fmt.Sprintf(":sparkles: **AI summary:** %v", aiSummary) + summary = fmt.Sprintf(":sparkles: **AI summary (experimental):** %v", aiSummary) } return summary, nil @@ -104,7 +104,7 @@ func GenerateChecksSummaryForJob( job *models.DiggerJob) (string, error) { } if aiSummary != "FOUR_OH_FOUR" { - summary += fmt.Sprintf(":sparkles: **AI summary:** %v", aiSummary) + summary += fmt.Sprintf(":sparkles: **AI summary (experimental):** %v", aiSummary) } return summary, nil diff --git a/backend/models/scheduler_test.go b/backend/models/scheduler_test.go index 892a95e85..46c25134d 100644 --- a/backend/models/scheduler_test.go +++ b/backend/models/scheduler_test.go @@ -75,7 +75,7 @@ func TestCreateDiggerJob(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy", "") assert.NoError(t, err) assert.NotNil(t, job) @@ -87,7 +87,7 @@ func TestCreateSingleJob(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy", "") assert.NoError(t, err) assert.NotNil(t, job) @@ -99,20 +99,20 @@ func TestFindDiggerJobsByParentJobId(t *testing.T) { defer teardownSuite(t) batchId, _ := uuid.NewUUID() - job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") + job, err := database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy", "") parentJobId := job.DiggerJobID assert.NoError(t, err) assert.NotNil(t, job) assert.NotZero(t, job.ID) - job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") + job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy", "") assert.NoError(t, err) assert.NotNil(t, job) assert.NotZero(t, job.ID) err = database.CreateDiggerJobParentLink(parentJobId, job.DiggerJobID) assert.Nil(t, err) - job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy") + job, err = database.CreateDiggerJob(batchId, []byte{100}, "digger_workflow.yml", nil, nil, "lazy", "") assert.NoError(t, err) assert.NotNil(t, job) err = database.CreateDiggerJobParentLink(parentJobId, job.DiggerJobID) diff --git a/backend/models/storage.go b/backend/models/storage.go index da7fcb7b0..066636c57 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -960,7 +960,7 @@ func (db *Database) UpdateBatchStatus(batch *DiggerBatch) error { return nil } -func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, workflowFile string, checkRunId *string, checkRunUrl *string, reporterType string) (*DiggerJob, error) { +func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, workflowFile string, checkRunId *string, checkRunUrl *string, reporterType string, projectName string) (*DiggerJob, error) { if serializedJob == nil || len(serializedJob) == 0 { return nil, fmt.Errorf("serializedJob can't be empty") } @@ -977,6 +977,7 @@ func (db *Database) CreateDiggerJob(batchId uuid.UUID, serializedJob []byte, wor job := &DiggerJob{ DiggerJobID: jobId, Status: scheduler.DiggerJobCreated, + ProjectName: projectName, BatchID: &batchIdStr, CheckRunId: checkRunId, CheckRunUrl: checkRunUrl, diff --git a/backend/models/storage_test.go b/backend/models/storage_test.go index 0e96e3fc5..c2e1f8301 100644 --- a/backend/models/storage_test.go +++ b/backend/models/storage_test.go @@ -146,7 +146,7 @@ func TestGetDiggerJobsForBatchPreloadsSummary(t *testing.T) { batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false, true, nil, "", nil, nil) assert.NoError(t, err) - job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml", nil, nil, "lazy") + job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml", nil, nil, "lazy", "") assert.NoError(t, err) job, err = DB.UpdateDiggerJobSummary(job.DiggerJobID, resourcesCreated, resourcesUpdated, resourcesDeleted) diff --git a/backend/utils/comment_utils.go b/backend/utils/comment_utils.go index 787c302cf..8a793b52d 100644 --- a/backend/utils/comment_utils.go +++ b/backend/utils/comment_utils.go @@ -55,6 +55,11 @@ func UpdatePRCommentRealtime(gh GithubClientProvider, batch *models.DiggerBatch) return fmt.Errorf("error requerying jobs for batch: %v", err) } + if freshBatch.CommentId == nil { + slog.Debug("No comment id found for batch, not updating", "batchId", batch.ID) + return nil + } + if len(freshJobs) == 0 { slog.Debug("No jobs found after requery", "batchId", freshBatch.ID) return nil diff --git a/backend/utils/github.go b/backend/utils/github.go index a0d1a4faa..de8fcfc5d 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -242,9 +242,20 @@ func SetPRCommitStatusForJobs(prService ci.PullRequestService, prNumber int, job return nil } +func GetCheckDetailedUrl(checkRunId int64, repoOwner string, repoName string, prNumber int) string { + githubHostname := os.Getenv("DIGGER_GITHUB_HOSTNAME") + if githubHostname == "" { + githubHostname = "github.com" + } + url := fmt.Sprintf( + "https://%v/%s/%s/pull/%d/checks?check_run_id=%d", githubHostname, repoOwner, repoName, prNumber, checkRunId, + ) + return url +} + // Checks are the more modern github way as opposed to "commit status" // With checks you also get to set a page representing content of the check -func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []scheduler.Job, commitSha string) (*CheckRunData, map[string]CheckRunData, error) { +func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []scheduler.Job, commitSha string, repoName string, repoOwner string) (*CheckRunData, map[string]CheckRunData, error) { slog.Info("commitSha", "commitsha", commitSha) slog.Info("Setting PR status for jobs", "prNumber", prNumber, @@ -268,7 +279,7 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/plan", "in_progress", "", "Waiting for plan...", "", "Plan result will appear here", commitSha, actions) jobCheckRunIds[job.ProjectName] = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), - Url: *cr.HTMLURL, + Url: GetCheckDetailedUrl(*cr.ID, repoOwner, repoName, prNumber, ), } case "digger apply": @@ -279,7 +290,7 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc cr, err = ghService.CreateCheckRun(job.GetProjectAlias()+"/apply", "in_progress", "", "Waiting for apply...", "", "Apply result will appear here", commitSha, nil) jobCheckRunIds[job.ProjectName] = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), - Url: *cr.URL, + Url: GetCheckDetailedUrl(*cr.ID, repoOwner, repoName, prNumber, ), } } if err != nil { @@ -304,14 +315,14 @@ func SetPRCheckForJobs(ghService *github2.GithubService, prNumber int, jobs []sc cr, err = ghService.CreateCheckRun("digger/plan", "in_progress", "", "Pending start...", "", jobsSummaryTable, commitSha, nil) batchCheckRunId = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), - Url: *cr.HTMLURL, + Url: GetCheckDetailedUrl(*cr.ID, repoOwner, repoName, prNumber, ), } } else { slog.Debug("Setting aggregate apply status", "prNumber", prNumber) cr, err = ghService.CreateCheckRun("digger/apply", "in_progress", "", "Pending start...", "", jobsSummaryTable, commitSha, nil) batchCheckRunId = CheckRunData{ Id: strconv.FormatInt(*cr.ID, 10), - Url: *cr.HTMLURL, + Url: GetCheckDetailedUrl(*cr.ID, repoOwner, repoName, prNumber, ), } } if err != nil { @@ -359,11 +370,18 @@ func GetActionsForBatch(batch *models.DiggerBatch) []*github.CheckRunAction { func GetActionsForJob(job *models.DiggerJob) []*github.CheckRunAction { batchActions := make([]*github.CheckRunAction, 0) if job.Status == scheduler.DiggerJobSucceeded { + batch := job.Batch batchActions = append(batchActions, &github.CheckRunAction{ - Label: "Apply job", // max 20 chars - Description: "Apply this job", // max 40 chars - Identifier: fmt.Sprintf("%v:%v", CheckedRunActionJobApply, job.DiggerJobID), // max 20 chars + Label: "Apply all", // max 20 chars + Description: "Apply all jobs", // max 40 chars + Identifier: fmt.Sprintf("%v:%v", CheckedRunActionBatchApply, batch.DiggerBatchID), // max 20 chars }) + // TODO: in the future when we support "apply single job we can add this + //batchActions = append(batchActions, &github.CheckRunAction{ + // Label: "Apply job", // max 20 chars + // Description: "Apply this job", // max 40 chars + // Identifier: fmt.Sprintf("%v:%v", CheckedRunActionJobApply, job.DiggerJobID), // max 20 chars + //}) } return batchActions } diff --git a/backend/utils/graphs.go b/backend/utils/graphs.go index e933fa509..69d043de6 100644 --- a/backend/utils/graphs.go +++ b/backend/utils/graphs.go @@ -15,7 +15,7 @@ import ( ) // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value -func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, jobReporterType string, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool, VCSConnectionId *uint, batchCheckRunData *CheckRunData, jobsCheckRunIdsMap map[string]CheckRunData) (*uuid.UUID, map[string]*models.DiggerJob, error) { +func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, jobReporterType string, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId *int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool, VCSConnectionId *uint, batchCheckRunData *CheckRunData, jobsCheckRunIdsMap map[string]CheckRunData) (*uuid.UUID, map[string]*models.DiggerJob, error) { slog.Info("Converting jobs to Digger jobs", "jobType", jobType, "vcsType", vcsType, @@ -78,7 +78,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, jobReporterType st batchCheckRunId = &batchCheckRunData.Id batchCheckRunUrl = &batchCheckRunData.Url } - batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput, coverAllImpactedProjects, VCSConnectionId, commitSha, batchCheckRunId, batchCheckRunUrl) + batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput, coverAllImpactedProjects, VCSConnectionId, commitSha, batchCheckRunId, batchCheckRunUrl) if err != nil { slog.Error("Failed to create batch", "error", err) return nil, nil, fmt.Errorf("failed to create batch: %v", err) @@ -109,7 +109,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, jobReporterType st } if predecessorMap[value] == nil || len(predecessorMap[value]) == 0 { slog.Debug("Processing node with no parents", "projectName", value) - parentJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl, jobReporterType) + parentJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl, jobReporterType, value) if err != nil { slog.Error("Failed to create job", "projectName", value, @@ -146,7 +146,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, jobReporterType st parent := edge.Source parentDiggerJob := result[parent] - childJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl, jobReporterType) + childJob, err := models.DB.CreateDiggerJob(batch.ID, marshalledJobsMap[value], projectMap[value].WorkflowFile, jobCheckRunId, jobCheckRunUrl, jobReporterType, value) if err != nil { slog.Error("Failed to create child job", "projectName", value, diff --git a/ee/backend/controllers/bitbucket.go b/ee/backend/controllers/bitbucket.go index f3320219d..00528b669 100644 --- a/ee/backend/controllers/bitbucket.go +++ b/ee/backend/controllers/bitbucket.go @@ -305,7 +305,7 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false, true, vcsConnectionId, nil, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, &commentId64, diggerYmlStr, 0, "", false, true, vcsConnectionId, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/ee/backend/controllers/gitlab.go b/ee/backend/controllers/gitlab.go index d7219b90b..c4ee20151 100644 --- a/ee/backend/controllers/gitlab.go +++ b/ee/backend/controllers/gitlab.go @@ -288,7 +288,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, &commentId, diggeryamlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -488,7 +488,7 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, &commentId64, diggerYmlStr, projectId, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/ee/backend/hooks/github.go b/ee/backend/hooks/github.go index 3200dee5d..607744d1e 100644 --- a/ee/backend/hooks/github.go +++ b/ee/backend/hooks/github.go @@ -71,7 +71,7 @@ var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.Git return nil } - diggerYmlStr, ghService, config, projectsGraph, err := ce_controllers.GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, defaultBranch, nil, nil) + diggerYmlStr, ghService, config, projectsGraph, err := ce_controllers.GetDiggerConfigForBranchOrSha(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, defaultBranch, "", nil, nil) if err != nil { log.Printf("Error loading digger.yml: %v", err) return fmt.Errorf("error loading digger.yml") @@ -152,7 +152,7 @@ var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.Git utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false, coverAllImpactedProjects, nil, nil, nil) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "lazy", "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", &reporterCommentId, diggerYmlStr, 0, "", false, coverAllImpactedProjects, nil, nil, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))