support caching of impacted projects

backend/controllers/github_comment.go

@@ -320,6 +320,21 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		"jobCount", len(jobs),
 	)
 
+
+	// impacted projects should have already been populated in the database by here since the PR open would have
+	// populated them, but just in case (disabled pr events or a long old pr before this change was deployed)
+	// we will populate them if they did not exist
+	dbImpactedProjects, err := models.DB.GetImpactedProjects(repoFullName, *commitSha)
+	if len(dbImpactedProjects) == 0 {
+		for _, impactedProject := range allImpactedProjects {
+			_, err = models.DB.CreateImpactedProject(repoFullName, *commitSha, impactedProject.Name, &prBranchName, &issueNumber)
+			if err != nil {
+				commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error failed to update internal record of impacted projects %v", err))
+				return err
+			}
+		}
+	}
+
 	// if flag set we dont allow more projects impacted than the number of changed files in PR (safety check)
 	if config2.LimitByNumOfFilesChanged() {
 		if len(impactedProjectsForComment) > len(changedFiles) {

backend/controllers/github_pull_request.go

@@ -204,6 +204,18 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		return nil
 	}
 
+	dbImpactedProjects, err := models.DB.GetImpactedProjects(repoFullName, commitSha)
+	// TODO: is this check for db impacted projects necessary?
+	if len(dbImpactedProjects) == 0 {
+		for _, impactedProject := range impactedProjects {
+			_, err = models.DB.CreateImpactedProject(repoFullName, commitSha, impactedProject.Name, &branch, &prNumber)
+			if err != nil {
+				commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error failed to update internal record of impacted projects %v", err))
+				return err
+			}
+		}
+	}
+
 	// if flag set we dont allow more projects impacted than the number of changed files in PR (safety check)
 	if config2.LimitByNumOfFilesChanged() {
 		if len(impactedProjects) > len(changedFiles) {

backend/controllers/projects.go

@@ -761,6 +761,22 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
 			)
 		}
 
+		commitSha := batch.CommitSha
+		impactedProjectDb, err := models.DB.GetImpactedProjectSingle(batch.RepoFullName, commitSha, job.ProjectName)
+		if err != nil {
+			slog.Warn("Error fetching impacted project db", "jobId", jobId, "error", err, "commitSha", commitSha, "repoFullName", batch.RepoFullName)
+		} else if impactedProjectDb == nil && err == nil {
+			slog.Warn("Impacted project entry not found in db (maybe it was not synced in event start)", "jobId", jobId, "error", err, "commitSha", commitSha, "repoFullName", batch.RepoFullName)
+		} else {
+			if batch.BatchType == orchestrator_scheduler.DiggerCommandPlan {
+				impactedProjectDb.Planned = true
+			}
+			if batch.BatchType == orchestrator_scheduler.DiggerCommandApply {
+				impactedProjectDb.Applied = true
+			}
+			models.DB.GormDB.Save(impactedProjectDb)
+		}
+
 		var prCommentId *int64
 		num, err := strconv.ParseInt(request.PrCommentId, 10, 64)
 		if err != nil {
@@ -1492,9 +1508,16 @@ func AutomergePRforBatchIfEnabled(gh utils.GithubClientProvider, batch *models.D
 		"batchType", batch.BatchType,
 	)
 
+	allApplied, _, err := models.DB.AllImpactedProjectApplied(batch.RepoFullName, batch.CommitSha)
+	if err != nil {
+		slog.Error("Error fetching all applied projects", "batchId", batch.ID, "error", err)
+		slog.Warn("falling back to using batch entry")
+		allApplied = batch.CoverAllImpactedProjects
+	}
+
 	if batch.Status == orchestrator_scheduler.BatchJobSucceeded &&
 		batch.BatchType == orchestrator_scheduler.DiggerCommandApply &&
-		batch.CoverAllImpactedProjects == true &&
+		allApplied &&
 		automerge == true {
 
 		slog.Info("Conditions met for auto-merge, proceeding",

backend/go.mod

@@ -18,7 +18,7 @@ replace github.com/diggerhq/digger/drift => ../drift
 replace github.com/ugorji/go => github.com/ugorji/go v1.2.12
 
 require (
-	ariga.io/atlas-provider-gorm v0.5.0
+	ariga.io/atlas-provider-gorm v0.5.4
 	github.com/bradleyfalzon/ghinstallation/v2 v2.16.0
 	github.com/dchest/uniuri v1.2.0
 	github.com/dominikbraun/graph v0.23.0
@@ -44,7 +44,7 @@ require (
 	golang.org/x/text v0.28.0
 	gorm.io/driver/postgres v1.6.0
 	gorm.io/driver/sqlite v1.6.0
-	gorm.io/gorm v1.31.0
+	gorm.io/gorm v1.30.1
 	k8s.io/api v0.20.6
 	k8s.io/apimachinery v0.20.6
 	k8s.io/client-go v10.0.0+incompatible

backend/go.sum

@@ -2,6 +2,8 @@ ariga.io/atlas-go-sdk v0.7.2 h1:pvS8tKVeRQuqdETBqj5qAQtVbQE88Gya6bOfY8YF3vU=
 ariga.io/atlas-go-sdk v0.7.2/go.mod h1:cFq7bnvHgKTWHCsU46mtkGxdl41rx2o7SjaLoh6cO8M=
 ariga.io/atlas-provider-gorm v0.5.0 h1:DqYNWroKUiXmx2N6nf/I9lIWu6fpgB6OQx/JoelCTes=
 ariga.io/atlas-provider-gorm v0.5.0/go.mod h1:8m6+N6+IgWMzPcR63c9sNOBoxfNk6yV6txBZBrgLg1o=
+ariga.io/atlas-provider-gorm v0.5.4 h1:64xboUDrP+JHdZOy4juPydHT5UP1kY152b5Gh/xNzmM=
+ariga.io/atlas-provider-gorm v0.5.4/go.mod h1:cXt4kxq8KIldPXHoWXC0HvSr8dVI0dIykZt3MZ4AmqE=
 c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3IqwfuN5kgDfo5MLzpNM0=
 c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805/go.mod h1:FomMrUJ2Lxt5jCLmZkG3FHa72zUprnhd3v/Z18Snm4w=
 cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4=
@@ -757,6 +759,10 @@ github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm
 github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
+github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
+github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
+github.com/alecthomas/kong v1.9.0 h1:Wgg0ll5Ys7xDnpgYBuBn/wPeLGAuK0NvYmEcisJgrIs=
+github.com/alecthomas/kong v1.9.0/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -2839,9 +2845,10 @@ gorm.io/gorm v1.23.7/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gorm.io/gorm v1.23.10/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
 gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
+gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
 gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
-gorm.io/gorm v1.31.0 h1:0VlycGreVhK7RF/Bwt51Fk8v0xLiiiFdbGDPIZQ7mJY=
-gorm.io/gorm v1.31.0/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
+gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4=
+gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
 gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=

backend/migrations/20251114205312.sql

@@ -0,0 +1,27 @@
+-- Modify "digger_batches" table
+ALTER TABLE "public"."digger_batches" ADD COLUMN "commit_sha" text NULL;
+-- Drop index "idx_ddr_deleted_at" from table: "digger_detection_runs"
+DROP INDEX "public"."idx_ddr_deleted_at";
+-- Drop index "idx_ddr_org_repo_pr_created_at" from table: "digger_detection_runs"
+DROP INDEX "public"."idx_ddr_org_repo_pr_created_at";
+-- Modify "digger_detection_runs" table
+ALTER TABLE "public"."digger_detection_runs" ALTER COLUMN "created_at" DROP NOT NULL, ALTER COLUMN "created_at" DROP DEFAULT;
+-- Create index "idx_ddr_org_repo_pr_created_at" to table: "digger_detection_runs"
+CREATE INDEX "idx_ddr_org_repo_pr_created_at" ON "public"."digger_detection_runs" ("organisation_id", "repo_full_name", "pr_number");
+-- Create index "idx_digger_detection_runs_deleted_at" to table: "digger_detection_runs"
+CREATE INDEX "idx_digger_detection_runs_deleted_at" ON "public"."digger_detection_runs" ("deleted_at");
+-- Create "impacted_projects" table
+CREATE TABLE "public"."impacted_projects" (
+  "id" text NOT NULL,
+  "created_at" timestamptz NULL,
+  "updated_at" timestamptz NULL,
+  "deleted_at" timestamptz NULL,
+  "repo_full_name" text NULL,
+  "commit_sha" text NULL,
+  "project_name" text NULL,
+  "planned" boolean NULL,
+  "applied" boolean NULL,
+  PRIMARY KEY ("id")
+);
+-- Create index "idx_impacted_projects_deleted_at" to table: "impacted_projects"
+CREATE INDEX "idx_impacted_projects_deleted_at" ON "public"."impacted_projects" ("deleted_at");

backend/migrations/20251114230419.sql

@@ -0,0 +1,2 @@
+-- Modify "impacted_projects" table
+ALTER TABLE "public"."impacted_projects" ADD COLUMN "pr_number" bigint NULL, ADD COLUMN "branch" text NULL;

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:Bodw0wkkaLj7PUoyJBn2J/C1a0k3CEZzYZVP7A4930g=
+h1:HUFvDp6jvx9L32hqunXbuvbX3cWvhFy8oVuq3SZ7xzY=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -67,3 +67,5 @@ h1:Bodw0wkkaLj7PUoyJBn2J/C1a0k3CEZzYZVP7A4930g=
 20250910102133.sql h1:jBW3PuoCWZPJA8ZaXDAyRuA9LnGDQGxvL+HtjCn33DI=
 20251006225238.sql h1:L581xAn5IsYt9Srf1RnJLleLIQVlgLzp7FaAChAlCJw=
 20251107000100.sql h1:b3USfhlLulZ+6iL9a66Ddpy6uDcYmmyDGZLYzbEjuRA=
+20251114205312.sql h1:RBQdD8zLKavCEfZOW8S2r31QBC9ZznCjB1Tw4SDJQGg=
+20251114230419.sql h1:/WA7vp7SKgdfe3KHS65nbwE4RUyUmlUOxuQ8tZZ/FQI=